Compare commits

...

68 Commits

Author SHA1 Message Date
Isaac Abadi
a36761e96a Fixed frontend build error 2022-05-01 23:24:15 -04:00
Isaac Abadi
88c16d7195 All setIntervals on the frontend are now properly destroyed 2022-05-01 23:23:19 -04:00
Isaac Abadi
8a323f028d Fixed bug where subscription avatars were missing 2022-05-01 23:15:09 -04:00
Isaac Abadi
a68726e7cb Removed deprecated armhf.Dockerfile 2022-05-01 23:13:11 -04:00
Isaac Abadi
dab0b7a8b6 Updated Angular version in readme 2022-05-01 23:13:00 -04:00
Isaac Abadi
9f9054ed9d Removed secrets from docker-pr.yml 2022-05-01 22:59:55 -04:00
Isaac Abadi
4c06c430eb Converted docker-compose build to docker build for docker-pr GH action 2022-05-01 21:21:39 -04:00
Isaac Abadi
2981f843c3 Added docker build PR check 2022-05-01 21:11:21 -04:00
Isaac Abadi
3a48ff2d50 docker build and push action now uses secrets for DockerHub username, repo, and tag 2022-05-01 21:11:01 -04:00
Glassed Silver
ac2c3dc8a1 Merge pull request #588 from GlassedSilver/master
removing strict SSL from npm config
2022-05-02 03:00:39 +02:00
GlassedSilver
0abe252d1e we need to find a different build check solution 2022-05-02 02:59:25 +02:00
GlassedSilver
f5f00e1732 fix name 2022-05-02 02:12:10 +02:00
GlassedSilver
c309e41a91 Merge branch 'master' of https://github.com/GlassedSilver/YoutubeDL-Material into master 2022-05-02 02:09:44 +02:00
GlassedSilver
754d837059 adding docker-pr-check.yml 2022-05-02 02:09:37 +02:00
Glassed Silver
d5626f1dae Dockerfile: wget not needed 2022-05-01 23:29:51 +02:00
GlassedSilver
9c0733453a removing strict SSL from npm config 2022-05-01 23:00:01 +02:00
Glassed Silver
2a41028253 Update Dockerfile 2022-05-01 20:42:45 +02:00
Glassed Silver
67b2e480f8 Merge pull request #586 from dejan995/master
Clean up docker image
2022-05-01 19:38:39 +02:00
dejan.petrov@dapmn.com
2cdc1cee98 Fix for #585
Added the DEBIAN_FRONTEND=noninteractive variable to all stages. This should stop the build from failing.
Also added --no-install-recommends to install only the requested packages.
This might break stuff, but I'm not sure though.
2022-05-01 18:14:27 +02:00
dejan.petrov@dapmn.com
bd1ed2b705 Clean up docker image
Added some commands to clean up the image after apt-get does its thing.
It should shave off a couple of megabytes, nothing to big though.
2022-05-01 18:02:46 +02:00
Glassed Silver
33ca0f0817 Merge pull request #584 from GlassedSilver/master
wow that was a bunch of work, but...
2022-05-01 12:30:49 +02:00
GlassedSilver
d5ab0d7b96 I'm getting sleepy, why am I still pushing through 2022-05-01 11:54:19 +02:00
GlassedSilver
777aebe508 apparently we still need npm in the last stretch.. 2022-05-01 11:52:35 +02:00
GlassedSilver
efaecaa059 use yarn in apt installs instead of npm 2022-05-01 11:48:12 +02:00
GlassedSilver
39ddefab5c fix dependencies needed for our apt packages 2022-05-01 11:37:39 +02:00
GlassedSilver
60f2ab449f yea 2022-05-01 11:31:53 +02:00
GlassedSilver
958f80e200 the good? I learn a lot about Docker building 2022-05-01 11:28:34 +02:00
GlassedSilver
7aa5c1bf7f syyyyntax 2022-05-01 11:21:45 +02:00
GlassedSilver
3bcbe0d3e7 fix dependency node-gyp (>= 3.6.2~) needed 2022-05-01 11:04:59 +02:00
GlassedSilver
80fcecdaea it's a learning experience 2022-05-01 10:57:21 +02:00
GlassedSilver
0329cd9718 don't think we need to install curl twice lol 2022-05-01 10:51:20 +02:00
GlassedSilver
493e876a97 syntax fixes are fun 2022-05-01 10:48:27 +02:00
Glassed Silver
574edd74ab Merge pull request #583 from GlassedSilver/master
I did warn you I will test docker builds this way
2022-05-01 10:41:10 +02:00
GlassedSilver
fe91484f24 I did warn you I will test docker builds this way 2022-05-01 10:40:19 +02:00
Glassed Silver
dda6e40a42 Merge pull request #582 from GlassedSilver/master
fix docker-build.sh for ubuntu, what a ride
2022-05-01 10:16:03 +02:00
GlassedSilver
c0fb838931 fix docker-build.sh for ubuntu, what a ride 2022-05-01 10:11:32 +02:00
Glassed Silver
28924cc7a0 Merge pull request #581 from GlassedSilver/master
fix pipefail MIA in ubuntu without specifying bash
2022-05-01 09:36:27 +02:00
GlassedSilver
2527051eab fix pipefail MIA in ubuntu without specifying bash 2022-05-01 09:35:04 +02:00
Glassed Silver
fcf7d14f46 Merge pull request #580 from GlassedSilver/master
Fix for #480 - existing DLs still getting queued
2022-05-01 09:21:48 +02:00
GlassedSilver
0a8aba54d2 Fix for #480 - existing DLs still getting queued 2022-05-01 09:17:23 +02:00
Glassed Silver
2c6485acb2 Merge pull request #577 from GlassedSilver/master
Dockerfile uses Ubuntu 20.04, fix obtain ffmpeg
2022-05-01 09:14:13 +02:00
GlassedSilver
aea4f52267 revert postbuild.mjs file-extension change 2022-05-01 07:12:00 +02:00
GlassedSilver
5ac5fca482 adapt postbuild.mjs to postbuild.js 2022-05-01 06:37:12 +02:00
GlassedSilver
7874f1b71a curl is in fact missing in focal, my bad 2022-05-01 06:29:54 +02:00
GlassedSilver
960c545f37 Dockerfile uses Ubuntu 20.04, fix obtain ffmpeg 2022-05-01 05:14:31 +02:00
Isaac Abadi
5e3eb68b03 Fixed issue where setting sub downloads as 'fresh' was not working properly (#567) 2022-04-30 00:58:12 -04:00
Glassed Silver
4dd3b97515 Merge pull request #566 from GlassedSilver/master
Fixing DNS issues caused by outdated musl version
2022-04-26 06:39:26 +02:00
Glassed Silver
701066eec1 Merge pull request #562 from Tzahi12345/GlassedSilver-add-security-policy
Added Security Policy
2022-04-26 06:39:16 +02:00
GlassedSilver
7f61ccb5f5 Use fixed version of musl to fix DNS errors 2022-04-26 04:46:05 +02:00
Glassed Silver
4f227ca442 Delete extensions.json 2022-04-26 04:28:47 +02:00
Glassed Silver
666bd2057d Merge branch 'Tzahi12345:master' into master 2022-04-26 04:25:50 +02:00
Isaac Abadi
37c858f950 Revert "Updated ffmpeg link in docker-build.sh to use release builds"
This reverts commit 768ec59f30.
2022-04-24 06:16:43 -04:00
Isaac Abadi
ebb7f6a2b0 Revert "Fixed mangled ffmpeg link"
This reverts commit 48e46db071.
2022-04-24 06:16:02 -04:00
Isaac Abadi
48e46db071 Fixed mangled ffmpeg link 2022-04-24 05:51:04 -04:00
Isaac Abadi
768ec59f30 Updated ffmpeg link in docker-build.sh to use release builds 2022-04-24 05:49:09 -04:00
Glassed Silver
aa8f602856 Added Security Policy 2022-04-24 11:12:22 +02:00
Isaac Abadi
d5c1361e64 Fixed issue where roles were not properly initialized 2022-04-24 05:02:02 -04:00
Isaac Abadi
901a96aada Fixed issue where use_local_db may be out of sync when first starting youtubedl-material 2022-04-24 05:01:45 -04:00
Isaac Abadi
21507ee36d Updated methodology of calculating download progress to rely on fs.readdir instead of glob 2022-04-24 04:15:38 -04:00
Isaac Abadi
0585943d67 Fixed bug where task time was not properly set with values of 0
Fixed issue where time field was not properly populated in the schedule dialog
2022-04-24 04:10:27 -04:00
Isaac Abadi
0bc2193f25 Updated downloadFile API request 2022-04-23 21:41:39 -04:00
Isaac Abadi
f3398fce1a Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material 2022-04-23 21:20:51 -04:00
Glassed Silver
60e8973f52 Merge branch 'Tzahi12345:master' into master 2022-04-24 00:40:58 +02:00
Isaac Abadi
d94857b0a5 Rolled back passport update 2022-04-23 18:14:44 -04:00
Glassed Silver
5fda56d7af Merge pull request #560 from Tzahi12345/revert-556-dependabot/npm_and_yarn/async-2.6.4
Revert "Bump async from 2.6.3 to 2.6.4"
2022-04-23 23:53:29 +02:00
Glassed Silver
abb80b33f3 Revert "Bump async from 2.6.3 to 2.6.4" 2022-04-23 23:53:15 +02:00
Glassed Silver
9977340161 Merge pull request #558 from Tzahi12345/angular-13-update
Angular/dependencies updates
2022-04-23 08:44:27 +02:00
GlassedSilver
c008171850 add color picker to WS recs 2022-04-21 22:33:01 +02:00
22 changed files with 227 additions and 129 deletions

27
.github/workflows/docker-pr.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: docker-pr
on:
pull_request:
branches: [master]
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: Build docker images
run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

View File

@@ -21,7 +21,7 @@ jobs:
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
@@ -39,4 +39,8 @@ jobs:
file: ./Dockerfile
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: tzahi12345/youtubedl-material:nightly
# Defaults:
# DOCKERHUB_USERNAME : tzahi12345
# DOCKERHUB_REPO : youtubedl-material
# DOCKERHUB_MASTER_TAG: nightly
tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}:${{secrets.DOCKERHUB_MASTER_TAG}}

View File

@@ -1,14 +1,25 @@
FROM alpine:latest AS ffmpeg
FROM ubuntu:20.04 AS ffmpeg
ENV DEBIAN_FRONTEND=noninteractive
COPY docker-build.sh .
RUN sh ./docker-build.sh
FROM alpine:latest as frontend
FROM ubuntu:20.04 as frontend
RUN apk add --no-cache \
npm
RUN npm install -g @angular/cli
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get -y install \
curl \
gnupg && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
apt-get -y install \
nodejs \
# YARN: brings along npm, solves dependency conflicts,
# spares us this spaghetti approach: https://stackoverflow.com/a/60547197
yarn && \
apt-get install -f && \
npm config set strict-ssl false && \
npm install -g @angular/cli
WORKDIR /build
COPY [ "package.json", "package-lock.json", "/build/" ]
@@ -20,35 +31,41 @@ RUN npm run build
#--------------#
FROM alpine:latest
FROM ubuntu:20.04
ENV UID=1000 \
GID=1000 \
USER=youtube
USER=youtube \
NO_UPDATE_NOTIFIER=true
ENV NO_UPDATE_NOTIFIER=true
ENV DEBIAN_FRONTEND=noninteractive
RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
RUN apk add --no-cache \
RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
RUN apt-get update && apt-get -y install \
npm \
python2 \
python3 \
su-exec \
&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
atomicparsley
atomicparsley && \
apt-get install -f && \
apt-get autoremove --purge && \
apt-get autoremove && \
apt-get clean && \
rm -rf /var/lib/apt
WORKDIR /app
COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe
COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
ENV PM2_HOME=/app/pm2
RUN npm install pm2 -g
RUN npm install && chown -R $UID:$GID ./
RUN npm config set strict-ssl false && \
npm install pm2 -g && \
npm install && chown -R $UID:$GID ./
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
# ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "pm2-runtime", "pm2.config.js" ]

View File

@@ -16,7 +16,7 @@ paths:
- downloader
summary: Download video file
description: |-
Downloads a video file with the given URL. Will include global args if they exist.
Downloads a file with the given URL. Will include global args if they exist.
HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status.
@@ -41,7 +41,7 @@ paths:
post:
tags:
- downloader
summary: Download video file
summary: Generates arguments used to download file
description: Generates args, used for checking what args would run if you ran downloadFile
operationId: post-generateArgs
requestBody:

View File

@@ -6,7 +6,7 @@
[![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
[![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
Now with [Docker](#Docker) support!

21
SECURITY.md Normal file
View File

@@ -0,0 +1,21 @@
# Security Policy
## Supported Versions
Currently all work on this project goes into the nightly builds.
4.2's RELEASE build is now quite old and should be considered legacy.
We urge users to use the nightly releases, because the project
constantly sees fixes.
| Version | Supported |
| ------------- | ------------------ |
| 4.2 Nightlies | :white_check_mark: |
| 4.2 Release | :x: |
| < 4.2 | :x: |
## Reporting a Vulnerability
Please file an issue in our GitHub's repo, because this app
isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

View File

@@ -1,49 +0,0 @@
FROM alpine:3.12 as frontend
RUN apk add --no-cache \
npm \
curl
RUN npm install -g @angular/cli
WORKDIR /build
RUN curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
COPY [ "package.json", "package-lock.json", "/build/" ]
RUN npm install
COPY [ "angular.json", "tsconfig.json", "/build/" ]
COPY [ "src/", "/build/src/" ]
RUN ng build --prod
#--------------#
FROM arm32v7/alpine:3.12
COPY --from=frontend /build/qemu-arm-static /usr/bin
ENV UID=1000 \
GID=1000 \
USER=youtube
RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
RUN apk add --no-cache \
ffmpeg \
npm \
python2 \
su-exec \
&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
atomicparsley
WORKDIR /app
COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
RUN npm install && chown -R $UID:$GID ./
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "node", "app.js" ]

View File

@@ -803,7 +803,7 @@ app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
app.post('/api/downloadFile', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos
const url = req.body.url;
const type = req.body.type;
const type = req.body.type ? req.body.type : 'video';
const user_uid = req.isAuthenticated() ? req.user.uid : null;
const options = {
customArgs: req.body.customArgs,

View File

@@ -18,10 +18,19 @@ let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function() {
exports.initialize = function () {
/*************************
* Authentication module
************************/
if (db_api.database_initialized) {
setupRoles();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupRoles();
});
}
saltRounds = 10;
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -49,6 +58,41 @@ exports.initialize = function() {
}));
}
const setupRoles = async () => {
const required_roles = {
admin: {
permissions: [
'filemanager',
'settings',
'subscriptions',
'sharing',
'advanced_download',
'downloads_manager'
]
},
user: {
permissions: [
'filemanager',
'subscriptions',
'sharing'
]
}
}
const role_keys = Object.keys(required_roles);
for (let i = 0; i < role_keys.length; i++) {
const role_key = role_keys[i];
const role_in_db = await db_api.getRecord('roles', {key: role_key});
if (!role_in_db) {
// insert task metadata into table if missing
await db_api.insertRecordIntoTable('roles', {
key: role_key,
permissions: required_roles[role_key]['permissions']
});
}
}
}
exports.passport = require('passport');
exports.passport.serializeUser(function(user, done) {

View File

@@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => {
}
exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
if (using_local_db && !custom_connection_string) return;
const success = await exports._connectToDB(custom_connection_string);
if (success) return true;

View File

@@ -3,7 +3,6 @@ const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const glob = require('glob')
const Mutex = require('async-mutex').Mutex;
const youtubedl = require('youtube-dl');
@@ -583,20 +582,26 @@ async function checkDownloadPercent(download_uid) {
if (!resulting_file_size) return;
let sum_size = 0;
glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
files.forEach(async file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
for (let i = 0; i < files_to_check_for_progress.length; i++) {
const file_to_check_for_progress = files_to_check_for_progress[i];
const dir = path.dirname(file_to_check_for_progress);
if (!fs.existsSync(dir)) continue;
fs.readdir(dir, async (err, files) => {
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (!file.includes(path.basename(file_to_check_for_progress))) continue;
try {
const file_stats = fs.statSync(path.join(dir, file));
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {}
}
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
});
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
});
}
}
exports.generateNFOFile = (info, output_path) => {

View File

@@ -2567,9 +2567,9 @@
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
},
"passport": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.5.2.tgz",
"integrity": "sha512-w9n/Ot5I7orGD4y+7V3EFJCQEznE5RxHamUxcqLT2QoJY0f2JdN8GyHonYFvN0Vz+L6lUJfVhrk2aZz2LbuREw==",
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
"integrity": "sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==",
"requires": {
"passport-strategy": "1.x.x",
"pause": "0.0.1"

View File

@@ -40,7 +40,6 @@
"express": "^4.17.3",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
"jsonwebtoken": "^8.5.1",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
@@ -53,7 +52,7 @@
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"nodemon": "^2.0.7",
"passport": "^0.5.2",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",
"passport-ldapauth": "^3.0.1",

View File

@@ -141,6 +141,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists
// TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
if (await fs.pathExists(archive_file_path)) {
await fs.unlink(archive_file_path);
}
@@ -266,11 +267,17 @@ async function getVideosForSub(sub, user_uid = null) {
}
resolve(false);
} else if (output) {
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
}
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name);
resolve(true);
return;
}
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
@@ -294,14 +301,7 @@ async function getVideosForSub(sub, user_uid = null) {
}
resolve(files_to_download);
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
}
resolve(true);
}
}
});
}, err => {
logger.error(err);
@@ -380,7 +380,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
if (sub.type && sub.type === 'audio') {
archive_path = path.join(archive_dir, 'merged_audio.txt');
} else {
archive_path = path.join(archive_dir, 'merged_video.txt');
}
}
downloadConfig.push('--download-archive', archive_path);
}
@@ -473,22 +477,24 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
return true;
}
async function setFreshUploads(sub, user_uid) {
async function setFreshUploads(sub) {
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (current_date === video['upload_date'].replace(/-/g, '')) {
sub_files.forEach(async file => {
if (current_date === file['upload_date'].replace(/-/g, '')) {
// set upload as fresh
const video_uid = video['uid'];
await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
const file_uid = file['uid'];
await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
}
});
}
async function checkVideosForFreshUploads(sub, user_uid) {
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
await checkVideoIfBetterExists(video, sub, user_uid)
sub_files.forEach(async file => {
if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
await checkVideoIfBetterExists(file, sub, user_uid)
}
});
}
@@ -510,13 +516,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (output) {
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
}
});
}
}
});
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
}
// helper functions

View File

@@ -42,9 +42,9 @@ function scheduleJob(task_key, schedule) {
if (schedule['type'] === 'timestamp') {
converted_schedule = new Date(schedule['data']['timestamp']);
} else if (schedule['type'] === 'recurring') {
const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] ? schedule['data']['minute'] : null;
const dayOfWeek = schedule['data']['dayOfWeek'] != null ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] != null ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] != null ? schedule['data']['minute'] : null;
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
} else {
logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)

View File

@@ -4,8 +4,6 @@
# and also optimizing some code with this commit.
# xoxo :D
set -xeuo pipefail
case $(uname -m) in
x86_64)
ARCH=amd64;;
@@ -22,10 +20,24 @@ case $(uname -m) in
exit 1
esac
echo "Architecture: $ARCH"
wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
echo "(INFO) Architecture detected: $ARCH"
echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
apt-get update && apt-get -y install curl xz-utils
echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
curl -o ffmpeg.txz \
--connect-timeout 5 \
--max-time 10 \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \
"https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
mkdir /tmp/ffmpeg
tar xf ffmpeg.txz -C /tmp/ffmpeg
echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
apt-get -y remove curl xz-utils
apt-get -y autoremove
echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
rm -rf /tmp/ffmpeg ffmpeg.txz

6
package-lock.json generated
View File

@@ -3367,9 +3367,9 @@
"dev": true
},
"async": {
"version": "2.6.4",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
"integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
"integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
"dev": true,
"requires": {
"lodash": "^4.17.14"

View File

@@ -51,7 +51,7 @@
</ng-container>
<ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
<mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
<a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatar [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatar>{{subscription.name}}</a>
<a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
</ng-container>
</mat-nav-list>
</mat-sidenav>

View File

@@ -42,6 +42,11 @@ export class ConcurrentStreamComponent implements OnInit {
}
ngOnDestroy(): void {
if (this.check_timeout) { clearInterval(this.check_timeout); }
if (this.update_timeout) { clearInterval(this.update_timeout); }
}
startServer() {
this.started = true;
this.server_started = true;

View File

@@ -1,4 +1,4 @@
import { AfterViewInit, Component, ElementRef, Input, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
import { Component, ElementRef, Input, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
import { PostsService } from 'app/posts.services';
@Component({
@@ -6,7 +6,7 @@ import { PostsService } from 'app/posts.services';
templateUrl: './twitch-chat.component.html',
styleUrls: ['./twitch-chat.component.scss']
})
export class TwitchChatComponent implements OnInit, AfterViewInit {
export class TwitchChatComponent implements OnInit, OnDestroy {
full_chat = null;
visible_chat = null;
@@ -33,7 +33,8 @@ export class TwitchChatComponent implements OnInit, AfterViewInit {
this.getFullChat();
}
ngAfterViewInit() {
ngOnDestroy(): void {
if (this.chat_check_interval_obj) { clearInterval(this.chat_check_interval_obj); }
}
private isUserNearBottom(): boolean {

View File

@@ -1,6 +1,6 @@
import { Component, Inject, OnInit } from '@angular/core';
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { Schedule } from 'api-types';
import { Schedule, Task } from 'api-types';
import { PostsService } from 'app/posts.services';
@Component({
@@ -18,7 +18,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
date = null;
today = new Date();
constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
constructor(@Inject(MAT_DIALOG_DATA) public data: {task: Task}, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
this.processTask(this.data.task);
this.postsService.getTask(this.data.task.key).subscribe(res => {
this.processTask(res['task']);
@@ -28,7 +28,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
ngOnInit(): void {
}
processTask(task) {
processTask(task: Task): void {
if (!task['schedule']) {
this.enabled = false;
return;
@@ -39,7 +39,11 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
this.recurring = schedule['type'] === Schedule.type.RECURRING;
if (this.recurring) {
this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`;
const hour = schedule['data']['hour'];
const minute = schedule['data']['minute'];
// add padding 0s if necessary to hours and minutes
this.time = (hour < 10 ? '0' : '') + hour + ':' + (minute < 10 ? '0' : '') + minute;
if (schedule['data']['dayOfWeek']) {
this.days_of_week = schedule['data']['dayOfWeek'];
@@ -75,7 +79,6 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
}
} else {
this.date.setHours(hours, minutes);
console.log(this.date);
schedule['data'] = {timestamp: this.date.getTime()};
}
this.dialogRef.close(schedule);

View File

@@ -52,6 +52,8 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
this.route.params.subscribe(params => {
this.id = params['id'];
if (this.sub_interval) { clearInterval(this.sub_interval); }
this.postsService.service_initialized.subscribe(init => {
if (init) {
this.getConfig();