From d94857b0a5396068c2c19b49c94d482ca5df12a6 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 23 Apr 2022 18:14:44 -0400 Subject: [PATCH 01/10] Rolled back passport update --- backend/package-lock.json | 6 +++--- backend/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 4bcad45..421898c 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -2567,9 +2567,9 @@ "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "passport": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/passport/-/passport-0.5.2.tgz", - "integrity": "sha512-w9n/Ot5I7orGD4y+7V3EFJCQEznE5RxHamUxcqLT2QoJY0f2JdN8GyHonYFvN0Vz+L6lUJfVhrk2aZz2LbuREw==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz", + "integrity": "sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==", "requires": { "passport-strategy": "1.x.x", "pause": "0.0.1" diff --git a/backend/package.json b/backend/package.json index 4c0b024..7d19c55 100644 --- a/backend/package.json +++ b/backend/package.json @@ -53,7 +53,7 @@ "node-id3": "^0.1.14", "node-schedule": "^2.1.0", "nodemon": "^2.0.7", - "passport": "^0.5.2", + "passport": "^0.4.1", "passport-http": "^0.3.0", "passport-jwt": "^4.0.0", "passport-ldapauth": "^3.0.1", From 0bc2193f2512ec7bdb3ebdc5be5006802cbd121d Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 23 Apr 2022 21:41:39 -0400 Subject: [PATCH 02/10] Updated downloadFile API request --- Public API v1.yaml | 4 ++-- backend/app.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Public API v1.yaml b/Public API v1.yaml index 10c5e3e..b1bccb7 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -16,7 +16,7 @@ paths: - downloader summary: Download video file description: |- - Downloads a video file with the given URL. Will include global args if they exist. + Downloads a file with the given URL. Will include global args if they exist. HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status. @@ -41,7 +41,7 @@ paths: post: tags: - downloader - summary: Download video file + summary: Generates arguments used to download file description: Generates args, used for checking what args would run if you ran downloadFile operationId: post-generateArgs requestBody: diff --git a/backend/app.js b/backend/app.js index 5881a56..2ec7f98 100644 --- a/backend/app.js +++ b/backend/app.js @@ -803,7 +803,7 @@ app.post('/api/testConnectionString', optionalJwt, async (req, res) => { app.post('/api/downloadFile', optionalJwt, async function(req, res) { req.setTimeout(0); // remove timeout in case of long videos const url = req.body.url; - const type = req.body.type; + const type = req.body.type ? req.body.type : 'video'; const user_uid = req.isAuthenticated() ? req.user.uid : null; const options = { customArgs: req.body.customArgs, From 0585943d67f09a93ac9bf0def42aabfc927e848f Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 04:10:27 -0400 Subject: [PATCH 03/10] Fixed bug where task time was not properly set with values of 0 Fixed issue where time field was not properly populated in the schedule dialog --- backend/tasks.js | 6 +++--- .../update-task-schedule-dialog.component.ts | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/backend/tasks.js b/backend/tasks.js index 2014da3..ba6379c 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -42,9 +42,9 @@ function scheduleJob(task_key, schedule) { if (schedule['type'] === 'timestamp') { converted_schedule = new Date(schedule['data']['timestamp']); } else if (schedule['type'] === 'recurring') { - const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null; - const hour = schedule['data']['hour'] ? schedule['data']['hour'] : null; - const minute = schedule['data']['minute'] ? schedule['data']['minute'] : null; + const dayOfWeek = schedule['data']['dayOfWeek'] != null ? schedule['data']['dayOfWeek'] : null; + const hour = schedule['data']['hour'] != null ? schedule['data']['hour'] : null; + const minute = schedule['data']['minute'] != null ? schedule['data']['minute'] : null; converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute); } else { logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`) diff --git a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts index ef61cf4..2d31782 100644 --- a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts +++ b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts @@ -1,6 +1,6 @@ import { Component, Inject, OnInit } from '@angular/core'; import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { Schedule } from 'api-types'; +import { Schedule, Task } from 'api-types'; import { PostsService } from 'app/posts.services'; @Component({ @@ -18,7 +18,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit { date = null; today = new Date(); - constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef, private postsService: PostsService) { + constructor(@Inject(MAT_DIALOG_DATA) public data: {task: Task}, private dialogRef: MatDialogRef, private postsService: PostsService) { this.processTask(this.data.task); this.postsService.getTask(this.data.task.key).subscribe(res => { this.processTask(res['task']); @@ -28,7 +28,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit { ngOnInit(): void { } - processTask(task) { + processTask(task: Task): void { if (!task['schedule']) { this.enabled = false; return; @@ -39,7 +39,11 @@ export class UpdateTaskScheduleDialogComponent implements OnInit { this.recurring = schedule['type'] === Schedule.type.RECURRING; if (this.recurring) { - this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`; + const hour = schedule['data']['hour']; + const minute = schedule['data']['minute']; + + // add padding 0s if necessary to hours and minutes + this.time = (hour < 10 ? '0' : '') + hour + ':' + (minute < 10 ? '0' : '') + minute; if (schedule['data']['dayOfWeek']) { this.days_of_week = schedule['data']['dayOfWeek']; @@ -75,7 +79,6 @@ export class UpdateTaskScheduleDialogComponent implements OnInit { } } else { this.date.setHours(hours, minutes); - console.log(this.date); schedule['data'] = {timestamp: this.date.getTime()}; } this.dialogRef.close(schedule); From 21507ee36dc560064129505f4b78c64a360a2c77 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 04:15:38 -0400 Subject: [PATCH 04/10] Updated methodology of calculating download progress to rely on fs.readdir instead of glob --- backend/downloader.js | 31 ++++++++++++++++++------------- backend/package.json | 1 - 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/downloader.js b/backend/downloader.js index f139a1f..108501b 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -3,7 +3,6 @@ const { uuid } = require('uuidv4'); const path = require('path'); const mergeFiles = require('merge-files'); const NodeID3 = require('node-id3') -const glob = require('glob') const Mutex = require('async-mutex').Mutex; const youtubedl = require('youtube-dl'); @@ -583,20 +582,26 @@ async function checkDownloadPercent(download_uid) { if (!resulting_file_size) return; let sum_size = 0; - glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => { - files.forEach(async file => { - try { - const file_stats = fs.statSync(file); - if (file_stats && file_stats.size) { - sum_size += file_stats.size; - } - } catch (e) { - + for (let i = 0; i < files_to_check_for_progress.length; i++) { + const file_to_check_for_progress = files_to_check_for_progress[i]; + const dir = path.dirname(file_to_check_for_progress); + if (!fs.existsSync(dir)) continue; + fs.readdir(dir, async (err, files) => { + for (let j = 0; j < files.length; j++) { + const file = files[j]; + if (!file.includes(path.basename(file_to_check_for_progress))) continue; + try { + const file_stats = fs.statSync(path.join(dir, file)); + if (file_stats && file_stats.size) { + sum_size += file_stats.size; + } + } catch (e) {} } + + const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2); + await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete}); }); - const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2); - await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete}); - }); + } } exports.generateNFOFile = (info, output_path) => { diff --git a/backend/package.json b/backend/package.json index 7d19c55..bf926a1 100644 --- a/backend/package.json +++ b/backend/package.json @@ -40,7 +40,6 @@ "express": "^4.17.3", "fluent-ffmpeg": "^2.1.2", "fs-extra": "^9.0.0", - "glob": "^7.1.6", "jsonwebtoken": "^8.5.1", "lowdb": "^1.0.0", "md5": "^2.2.1", From 901a96aada207a20d53292c4e78656d032858e31 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 05:01:45 -0400 Subject: [PATCH 05/10] Fixed issue where use_local_db may be out of sync when first starting youtubedl-material --- backend/db.js | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/db.js b/backend/db.js index 591b92b..7cab07d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => { } exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => { + using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify if (using_local_db && !custom_connection_string) return; const success = await exports._connectToDB(custom_connection_string); if (success) return true; From d5c1361e64a207973ff82e92811c884d1fa3ed7c Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 05:02:02 -0400 Subject: [PATCH 06/10] Fixed issue where roles were not properly initialized --- backend/authentication/auth.js | 46 +++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index de54a0b..7607c9b 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -18,10 +18,19 @@ let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function() { +exports.initialize = function () { /************************* * Authentication module ************************/ + + if (db_api.database_initialized) { + setupRoles(); + } else { + db_api.database_initialized_bs.subscribe(init => { + if (init) setupRoles(); + }); + } + saltRounds = 10; JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration'); @@ -49,6 +58,41 @@ exports.initialize = function() { })); } +const setupRoles = async () => { + const required_roles = { + admin: { + permissions: [ + 'filemanager', + 'settings', + 'subscriptions', + 'sharing', + 'advanced_download', + 'downloads_manager' + ] + }, + user: { + permissions: [ + 'filemanager', + 'subscriptions', + 'sharing' + ] + } + } + + const role_keys = Object.keys(required_roles); + for (let i = 0; i < role_keys.length; i++) { + const role_key = role_keys[i]; + const role_in_db = await db_api.getRecord('roles', {key: role_key}); + if (!role_in_db) { + // insert task metadata into table if missing + await db_api.insertRecordIntoTable('roles', { + key: role_key, + permissions: required_roles[role_key]['permissions'] + }); + } + } +} + exports.passport = require('passport'); exports.passport.serializeUser(function(user, done) { From 768ec59f30df81f7d2e81e7fb696409e234c0dc3 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 05:49:09 -0400 Subject: [PATCH 07/10] Updated ffmpeg link in docker-build.sh to use release builds --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 5876372..1c0fe93 100644 --- a/docker-build.sh +++ b/docker-build.sh @@ -23,7 +23,7 @@ case $(uname -m) in esac echo "Architecture: $ARCH" -wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz +wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz mkdir /tmp/ffmpeg tar xf ffmpeg.txz -C /tmp/ffmpeg cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg From 48e46db071448005c6c7b65030ed6fafaf5e7194 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 05:51:04 -0400 Subject: [PATCH 08/10] Fixed mangled ffmpeg link --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 1c0fe93..9aa80e0 100644 --- a/docker-build.sh +++ b/docker-build.sh @@ -23,7 +23,7 @@ case $(uname -m) in esac echo "Architecture: $ARCH" -wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz +wget "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz mkdir /tmp/ffmpeg tar xf ffmpeg.txz -C /tmp/ffmpeg cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg From ebb7f6a2b0e49e21c9e1f1795ea38c2c543f5e42 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 06:16:02 -0400 Subject: [PATCH 09/10] Revert "Fixed mangled ffmpeg link" This reverts commit 48e46db071448005c6c7b65030ed6fafaf5e7194. --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 9aa80e0..1c0fe93 100644 --- a/docker-build.sh +++ b/docker-build.sh @@ -23,7 +23,7 @@ case $(uname -m) in esac echo "Architecture: $ARCH" -wget "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz +wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz mkdir /tmp/ffmpeg tar xf ffmpeg.txz -C /tmp/ffmpeg cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg From 37c858f950592c37bf4b60679802ff51513a386c Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 24 Apr 2022 06:16:43 -0400 Subject: [PATCH 10/10] Revert "Updated ffmpeg link in docker-build.sh to use release builds" This reverts commit 768ec59f30df81f7d2e81e7fb696409e234c0dc3. --- docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-build.sh b/docker-build.sh index 1c0fe93..5876372 100644 --- a/docker-build.sh +++ b/docker-build.sh @@ -23,7 +23,7 @@ case $(uname -m) in esac echo "Architecture: $ARCH" -wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-release-${ARCH}-static.tar.xz" -O ffmpeg.txz +wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz mkdir /tmp/ffmpeg tar xf ffmpeg.txz -C /tmp/ffmpeg cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg