From ea43ea7121e94de2e195a0faac942f323ad584e7 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 5 Sep 2023 00:35:36 -0400 Subject: [PATCH] Backend can kick off downloads without using deprecated node-youtube-dl library Downloads can now be cancelled and better "paused" --- Public API v1.yaml | 2 + backend/downloader.js | 54 +++++++++++++------ backend/package.json | 1 + backend/subscriptions.js | 6 +-- backend/test/tests.js | 10 ++++ backend/youtube-dl.js | 41 +++++++++++++- src/api-types/models/Download.ts | 1 + .../downloads/downloads.component.ts | 5 +- src/app/main/main.component.ts | 11 +++- 9 files changed, 105 insertions(+), 26 deletions(-) diff --git a/Public API v1.yaml b/Public API v1.yaml index e49783d..0cf9507 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -2683,6 +2683,8 @@ components: type: boolean paused: type: boolean + cancelled: + type: boolean finished_step: type: boolean url: diff --git a/backend/downloader.js b/backend/downloader.js index a7adc62..aa8b13f 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -4,8 +4,6 @@ const path = require('path'); const NodeID3 = require('node-id3') const Mutex = require('async-mutex').Mutex; -const youtubedl = require('youtube-dl'); - const logger = require('./logger'); const youtubedl_api = require('./youtube-dl'); const config_api = require('./config'); @@ -21,6 +19,8 @@ const archive_api = require('./archive'); const mutex = new Mutex(); let should_check_downloads = true; +const download_to_child_process = {}; + if (db_api.database_initialized) { exports.setupDownloads(); } else { @@ -84,8 +84,11 @@ exports.pauseDownload = async (download_uid) => { } else if (download['finished']) { logger.info(`Download ${download_uid} could not be paused before completing.`); return false; + } else { + logger.info(`Pausing download ${download_uid}`); } + killActiveDownload(download); return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false}); } @@ -120,16 +123,23 @@ exports.cancelDownload = async (download_uid) => { } else if (download['finished']) { logger.info(`Download ${download_uid} could not be cancelled before completing.`); return false; + } else { + logger.info(`Cancelling download ${download_uid}`); } - return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false}); + + killActiveDownload(download); + await handleDownloadError(download_uid, 'Cancelled', 'cancelled'); + return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true}); } exports.clearDownload = async (download_uid) => { return await db_api.removeRecord('download_queue', {uid: download_uid}); } -async function handleDownloadError(download, error_message, error_type = null) { - if (!download || !download['uid']) return; +async function handleDownloadError(download_uid, error_message, error_type = null) { + if (!download_uid) return; + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (download['error']) return; notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type); await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type}); } @@ -180,7 +190,7 @@ async function checkDownloads() { if (waiting_download['sub_id']) { const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']})); if (sub_missing) { - handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing'); + handleDownloadError(waiting_download['uid'], `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing'); continue; } } @@ -195,6 +205,14 @@ async function checkDownloads() { } } +function killActiveDownload(download) { + const child_process = download_to_child_process[download['uid']]; + if (download['step_index'] === 2 && child_process) { + youtubedl_api.killYoutubeDLProcess(child_process); + delete download_to_child_process[download['uid']]; + } +} + exports.collectInfo = async (download_uid) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { @@ -232,8 +250,7 @@ exports.collectInfo = async (download_uid) => { const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`; logger.warn(error); if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error, 'exists_in_archive'); + await handleDownloadError(download_uid, error, 'exists_in_archive'); return; } } @@ -276,7 +293,7 @@ exports.collectInfo = async (download_uid) => { }); } -exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => { +exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { return; @@ -306,21 +323,25 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000); const file_objs = []; // download file - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod); + let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod); + if (child_process) download_to_child_process[download['uid']] = child_process; + const {parsed_output, err} = await callback; clearInterval(download_checker); let end_time = Date.now(); let difference = (end_time - start_time)/1000; logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); if (!parsed_output) { - logger.error(err.stderr); - await handleDownloadError(download, err.stderr, 'unknown_error'); + const errored_download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (errored_download['paused']) return; + logger.error(err.toString()); + await handleDownloadError(download_uid, err.toString(), 'unknown_error'); resolve(false); return; } else if (parsed_output) { if (parsed_output.length === 0 || parsed_output[0].length === 0) { // ERROR! const error_message = `No output received for video download, check if it exists in your archive.`; - await handleDownloadError(download, error_message, 'no_output'); + await handleDownloadError(download_uid, error_message, 'no_output'); logger.warn(error_message); resolve(false); return; @@ -392,7 +413,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec } else { const error_message = 'Downloaded file failed to result in metadata object.'; logger.error(error_message); - await handleDownloadError(download, error_message, 'no_metadata'); + await handleDownloadError(download_uid, error_message, 'no_metadata'); } const file_uids = file_objs.map(file_obj => file_obj.uid); @@ -547,14 +568,13 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { new_args.push('--dump-json'); - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, new_args); + const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args); if (!parsed_output || parsed_output.length === 0) { let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; if (err.stderr) error_message += `\n\n${err.stderr}`; logger.error(error_message); if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error_message, 'info_retrieve_failed'); + await handleDownloadError(download_uid, error_message, 'info_retrieve_failed'); } return null; } diff --git a/backend/package.json b/backend/package.json index 11f0e92..3ca4270 100644 --- a/backend/package.json +++ b/backend/package.json @@ -61,6 +61,7 @@ "read-last-lines": "^1.7.2", "rxjs": "^7.3.0", "shortid": "^2.2.15", + "tree-kill": "^1.2.2", "unzipper": "^0.10.10", "uuidv4": "^6.2.13", "winston": "^3.7.2", diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 4d2c9da..8085f26 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -64,7 +64,7 @@ async function getSubscriptionInfo(sub) { } } - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); if (err) { logger.error(err.stderr); return false; @@ -226,7 +226,7 @@ exports.getVideosForSub = async (sub, user_uid = null) => { // get videos logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`); - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); updateSubscriptionProperty(sub, {downloading: false}, user_uid); if (!parsed_output) { logger.error('Subscription check failed!'); @@ -482,7 +482,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; if (output[metric_to_compare] > file_obj[metric_to_compare]) { // download new video as the simulated one is better - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); if (err) { logger.verbose(`Failed to download better version of video ${file_obj['id']}`); } else if (parsed_output) { diff --git a/backend/test/tests.js b/backend/test/tests.js index e1bd3b2..be61d55 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -650,6 +650,16 @@ describe('youtube-dl', async function() { } config_api.setConfigItem('ytdl_default_downloader', original_fork); }); + + it('Run process', async function() { + this.timeout(300000); + const downloader_api = require('../downloader'); + const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI'; + const args = await downloader_api.generateArgs(url, 'video', {}, null, true); + const {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args); + assert(child_process); + console.log(await callback); + }); }); describe('Tasks', function() { diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js index 6bea878..46177d8 100644 --- a/backend/youtube-dl.js +++ b/backend/youtube-dl.js @@ -1,5 +1,7 @@ const fs = require('fs-extra'); const fetch = require('node-fetch'); +const execa = require('execa'); +const kill = require('tree-kill'); const logger = require('./logger'); const utils = require('./utils'); @@ -24,7 +26,20 @@ exports.youtubedl_forks = { } } -exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => { +exports.runYoutubeDL = async (url, args, downloadMethod = null) => { + let callback = null; + let child_process = null; + if (downloadMethod) { + callback = exports.runYoutubeDLMain(url, args, downloadMethod); + } else { + ({callback, child_process} = await runYoutubeDLProcess(url, args)); + } + + return {child_process, callback}; +} + +// Run youtube-dl in a main thread (with possible downloadMethod) +exports.runYoutubeDLMain = async (url, args, downloadMethod = youtubedl.exec) => { return new Promise(resolve => { downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) { const parsed_output = utils.parseOutputJSON(output, err); @@ -33,6 +48,30 @@ exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => { }); } +// Run youtube-dl in a subprocess +const runYoutubeDLProcess = async (url, args) => { + const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity}); + const callback = new Promise(async resolve => { + try { + const {stdout, stderr} = await child_process; + const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr); + resolve({parsed_output, err: stderr}); + } catch (e) { + resolve({parsed_output: null, err: e}) + } + }); + return {child_process, callback} +} + +async function getYoutubeDLPath() { + const guessed_base_path = 'node_modules/youtube-dl/bin/'; + return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); +} + +exports.killYoutubeDLProcess = async (child_process) => { + kill(child_process.pid, 'SIGKILL'); +} + exports.checkForYoutubeDLUpdate = async () => { const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); // get current version diff --git a/src/api-types/models/Download.ts b/src/api-types/models/Download.ts index 84d95e5..52c3529 100644 --- a/src/api-types/models/Download.ts +++ b/src/api-types/models/Download.ts @@ -8,6 +8,7 @@ export type Download = { running: boolean; finished: boolean; paused: boolean; + cancelled?: boolean; finished_step: boolean; url: string; type: string; diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index d5e3092..9490e8d 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -69,8 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { tooltip: $localize`Pause`, action: (download: Download) => this.pauseDownload(download), show: (download: Download) => !download.finished && (!download.paused || !download.finished_step), - icon: 'pause', - loading: (download: Download) => download.paused && !download.finished_step + icon: 'pause' }, { tooltip: $localize`Resume`, @@ -81,7 +80,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { { tooltip: $localize`Cancel`, action: (download: Download) => this.cancelDownload(download), - show: (download: Download) => false && !download.finished && !download.paused, // TODO: add possibility to cancel download + show: (download: Download) => !download.finished && !download.paused && !download.cancelled, icon: 'cancel' }, { diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 731dbe6..3a4058b 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -169,6 +169,8 @@ export class MainComponent implements OnInit { argsChangedSubject: Subject = new Subject(); simulatedOutput = ''; + interval_id = null; + constructor(public postsService: PostsService, private youtubeSearch: YoutubeSearchService, public snackBar: MatSnackBar, private router: Router, public dialog: MatDialog, private platform: Platform, private route: ActivatedRoute) { this.audioOnly = false; @@ -232,11 +234,12 @@ export class MainComponent implements OnInit { } // get downloads routine - setInterval(() => { + if (this.interval_id) { clearInterval(this.interval_id) } + this.interval_id = setInterval(() => { if (this.current_download) { this.getCurrentDownload(); } - }, 500); + }, 1000); return true; } @@ -294,6 +297,10 @@ export class MainComponent implements OnInit { } } + ngOnDestroy(): void { + if (this.interval_id) { clearInterval(this.interval_id) } + } + // download helpers downloadHelper(container: DatabaseFile | Playlist, type: string, is_playlist = false, force_view = false, navigate_mode = false): void { this.downloadingfile = false;