Backend can kick off downloads without using deprecated node-youtube-dl library

Downloads can now be cancelled and better "paused"
2026-03-07 12:00:01 +03:00 · 2023-09-05 00:35:36 -04:00
parent 3fc1110fc1
commit ea43ea7121
9 changed files with 105 additions and 26 deletions
--- a/v1.yaml
+++ b/v1.yaml
@@ -2683,6 +2683,8 @@ components:
          type: boolean
        paused:
          type: boolean
+        cancelled:
+          type: boolean
        finished_step:
          type: boolean
        url:
--- a/backend/downloader.js
+++ b/backend/downloader.js
@@ -4,8 +4,6 @@ const path = require('path');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;

-const youtubedl = require('youtube-dl');
-
 const logger = require('./logger');
 const youtubedl_api = require('./youtube-dl');
 const config_api = require('./config');
@@ -21,6 +19,8 @@ const archive_api = require('./archive');
 const mutex = new Mutex();
 let should_check_downloads = true;

+const download_to_child_process = {};
+
 if (db_api.database_initialized) {
    exports.setupDownloads();
 } else {
@@ -84,8 +84,11 @@ exports.pauseDownload = async (download_uid) => {
    } else if (download['finished']) {
        logger.info(`Download ${download_uid} could not be paused before completing.`);
        return false;
+    } else {
+        logger.info(`Pausing download ${download_uid}`);
    }

+    killActiveDownload(download);
    return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
 }

@@ -120,16 +123,23 @@ exports.cancelDownload = async (download_uid) => {
    } else if (download['finished']) {
        logger.info(`Download ${download_uid} could not be cancelled before completing.`);
        return false;
+    } else {
+        logger.info(`Cancelling download ${download_uid}`);
    }
-    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
+
+    killActiveDownload(download);
+    await handleDownloadError(download_uid, 'Cancelled', 'cancelled');
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true});
 }

 exports.clearDownload = async (download_uid) => {
    return await db_api.removeRecord('download_queue', {uid: download_uid});
 }

-async function handleDownloadError(download, error_message, error_type = null) {
-    if (!download || !download['uid']) return;
+async function handleDownloadError(download_uid, error_message, error_type = null) {
+    if (!download_uid) return;
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['error']) return;
    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
@@ -180,7 +190,7 @@ async function checkDownloads() {
            if (waiting_download['sub_id']) {
                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
                if (sub_missing) {
-                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    handleDownloadError(waiting_download['uid'], `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
                    continue;
                }
            }
@@ -195,6 +205,14 @@ async function checkDownloads() {
    }
 }

+function killActiveDownload(download) {
+    const child_process = download_to_child_process[download['uid']];
+    if (download['step_index'] === 2 && child_process) {
+        youtubedl_api.killYoutubeDLProcess(child_process);
+        delete download_to_child_process[download['uid']];
+    }
+}
+
 exports.collectInfo = async (download_uid) => {
    const download = await db_api.getRecord('download_queue', {uid: download_uid});
    if (download['paused']) {
@@ -232,8 +250,7 @@ exports.collectInfo = async (download_uid) => {
            const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
            logger.warn(error);
            if (download_uid) {
-                const download = await db_api.getRecord('download_queue', {uid: download_uid});
-                await handleDownloadError(download, error, 'exists_in_archive');
+                await handleDownloadError(download_uid, error, 'exists_in_archive');
                return;
            }
        }
@@ -276,7 +293,7 @@ exports.collectInfo = async (download_uid) => {
                                                                });
 }

-exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => {
+exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
    const download = await db_api.getRecord('download_queue', {uid: download_uid});
    if (download['paused']) {
        return;
@@ -306,21 +323,25 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
        const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
        const file_objs = [];
        // download file
-        const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
+        let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
+        if (child_process) download_to_child_process[download['uid']] = child_process;
+        const {parsed_output, err} = await callback;
        clearInterval(download_checker);
        let end_time = Date.now();
        let difference = (end_time - start_time)/1000;
        logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
        if (!parsed_output) {
-            logger.error(err.stderr);
-            await handleDownloadError(download, err.stderr, 'unknown_error');
+            const errored_download = await db_api.getRecord('download_queue', {uid: download_uid});
+            if (errored_download['paused']) return;
+            logger.error(err.toString());
+            await handleDownloadError(download_uid, err.toString(), 'unknown_error');
            resolve(false);
            return;
        } else if (parsed_output) {
            if (parsed_output.length === 0 || parsed_output[0].length === 0) {
                // ERROR!
                const error_message = `No output received for video download, check if it exists in your archive.`;
-                await handleDownloadError(download, error_message, 'no_output');
+                await handleDownloadError(download_uid, error_message, 'no_output');
                logger.warn(error_message);
                resolve(false);
                return;
@@ -392,7 +413,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
            } else {
                const error_message = 'Downloaded file failed to result in metadata object.';
                logger.error(error_message);
-                await handleDownloadError(download, error_message, 'no_metadata');
+                await handleDownloadError(download_uid, error_message, 'no_metadata');
            }

            const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -547,14 +568,13 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {

    new_args.push('--dump-json');

-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, new_args);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args);
    if (!parsed_output || parsed_output.length === 0) {
        let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
        if (err.stderr) error_message += `\n\n${err.stderr}`;
        logger.error(error_message);
        if (download_uid) {
-            const download = await db_api.getRecord('download_queue', {uid: download_uid});
-            await handleDownloadError(download, error_message, 'info_retrieve_failed');
+            await handleDownloadError(download_uid, error_message, 'info_retrieve_failed');
        }
        return null;
    }
--- a/backend/package.json
+++ b/backend/package.json
@@ -61,6 +61,7 @@
    "read-last-lines": "^1.7.2",
    "rxjs": "^7.3.0",
    "shortid": "^2.2.15",
+    "tree-kill": "^1.2.2",
    "unzipper": "^0.10.10",
    "uuidv4": "^6.2.13",
    "winston": "^3.7.2",
--- a/backend/subscriptions.js
+++ b/backend/subscriptions.js
@@ -64,7 +64,7 @@ async function getSubscriptionInfo(sub) {
        }
    }

-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
    if (err) {
        logger.error(err.stderr);
        return false;
@@ -226,7 +226,7 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
    // get videos
    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);

-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
    updateSubscriptionProperty(sub, {downloading: false}, user_uid);
    if (!parsed_output) {
        logger.error('Subscription check failed!');
@@ -482,7 +482,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
            const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
            if (output[metric_to_compare] > file_obj[metric_to_compare]) {
                // download new video as the simulated one is better
-                const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+                const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
                if (err) {
                    logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                } else if (parsed_output) {
--- a/backend/test/tests.js
+++ b/backend/test/tests.js
@@ -650,6 +650,16 @@ describe('youtube-dl', async function() {
        }
        config_api.setConfigItem('ytdl_default_downloader', original_fork);
    });
+
+    it('Run process', async function() {
+        this.timeout(300000);
+        const downloader_api = require('../downloader');
+        const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
+        const args = await downloader_api.generateArgs(url, 'video', {}, null, true);
+        const {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args);
+        assert(child_process);
+        console.log(await callback);
+    });
 });

 describe('Tasks', function() {
--- a/backend/youtube-dl.js
+++ b/backend/youtube-dl.js
@@ -1,5 +1,7 @@
 const fs = require('fs-extra');
 const fetch = require('node-fetch');
+const execa = require('execa');
+const kill = require('tree-kill');

 const logger = require('./logger');
 const utils = require('./utils');
@@ -24,7 +26,20 @@ exports.youtubedl_forks = {
    }
 }

-exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
+exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
+    let callback = null;
+    let child_process = null;
+    if (downloadMethod) {
+        callback = exports.runYoutubeDLMain(url, args, downloadMethod);
+    } else {
+        ({callback, child_process} = await runYoutubeDLProcess(url, args));
+    }
+
+    return {child_process, callback};
+}
+
+// Run youtube-dl in a main thread (with possible downloadMethod)
+exports.runYoutubeDLMain = async (url, args, downloadMethod = youtubedl.exec) => {
    return new Promise(resolve => {
        downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
            const parsed_output = utils.parseOutputJSON(output, err);
@@ -33,6 +48,30 @@ exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
    });
 }

+// Run youtube-dl in a subprocess
+const runYoutubeDLProcess = async (url, args) => {
+    const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity});
+    const callback = new Promise(async resolve => {
+        try {
+            const {stdout, stderr} = await child_process;
+            const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr);
+            resolve({parsed_output, err: stderr});
+        } catch (e) {
+            resolve({parsed_output: null, err: e})
+        }
+    });
+    return {child_process, callback}
+}
+
+async function getYoutubeDLPath() {
+    const guessed_base_path = 'node_modules/youtube-dl/bin/';
+    return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
+}
+
+exports.killYoutubeDLProcess = async (child_process) => {
+    kill(child_process.pid, 'SIGKILL');
+}
+
 exports.checkForYoutubeDLUpdate = async () => {
    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
    // get current version
--- a/src/api-types/models/Download.ts
+++ b/src/api-types/models/Download.ts
@@ -8,6 +8,7 @@ export type Download = {
    running: boolean;
    finished: boolean;
    paused: boolean;
+    cancelled?: boolean;
    finished_step: boolean;
    url: string;
    type: string;
--- a/src/app/components/downloads/downloads.component.ts
+++ b/src/app/components/downloads/downloads.component.ts
@@ -69,8 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
      tooltip: $localize`Pause`,
      action: (download: Download) => this.pauseDownload(download),
      show: (download: Download) => !download.finished && (!download.paused || !download.finished_step),
-      icon: 'pause',
-      loading: (download: Download) => download.paused && !download.finished_step
+      icon: 'pause'
    },
    {
      tooltip: $localize`Resume`,
@@ -81,7 +80,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
    {
      tooltip: $localize`Cancel`,
      action: (download: Download) => this.cancelDownload(download),
-      show: (download: Download) => false && !download.finished && !download.paused, // TODO: add possibility to cancel download
+      show: (download: Download) => !download.finished && !download.paused && !download.cancelled,
      icon: 'cancel'
    },
    {
--- a/src/app/main/main.component.ts
+++ b/src/app/main/main.component.ts
@@ -169,6 +169,8 @@ export class MainComponent implements OnInit {
  argsChangedSubject: Subject<boolean> = new Subject<boolean>();
  simulatedOutput = '';

+  interval_id = null;
+
  constructor(public postsService: PostsService, private youtubeSearch: YoutubeSearchService, public snackBar: MatSnackBar,
    private router: Router, public dialog: MatDialog, private platform: Platform, private route: ActivatedRoute) {
    this.audioOnly = false;
@@ -232,11 +234,12 @@ export class MainComponent implements OnInit {
    }

    // get downloads routine
-    setInterval(() => {
+    if (this.interval_id) { clearInterval(this.interval_id) }
+    this.interval_id = setInterval(() => {
      if (this.current_download) {
        this.getCurrentDownload();
      }
-    }, 500);
+    }, 1000);

    return true;
  }
@@ -294,6 +297,10 @@ export class MainComponent implements OnInit {
    }
  }

+  ngOnDestroy(): void {
+    if (this.interval_id) { clearInterval(this.interval_id) }
+  }
+
  // download helpers
  downloadHelper(container: DatabaseFile | Playlist, type: string, is_playlist = false, force_view = false, navigate_mode = false): void {
    this.downloadingfile = false;