Backend can kick off downloads without using deprecated node-youtube-dl library

Downloads can now be cancelled and better "paused"

Public API v1.yaml

@@ -2683,6 +2683,8 @@ components:
           type: boolean
         paused:
           type: boolean
+        cancelled:
+          type: boolean
         finished_step:
           type: boolean
         url:

backend/downloader.js

@@ -4,8 +4,6 @@ const path = require('path');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
-const youtubedl = require('youtube-dl');
-
 const logger = require('./logger');
 const youtubedl_api = require('./youtube-dl');
 const config_api = require('./config');
@@ -21,6 +19,8 @@ const archive_api = require('./archive');
 const mutex = new Mutex();
 let should_check_downloads = true;
 
+const download_to_child_process = {};
+
 if (db_api.database_initialized) {
     exports.setupDownloads();
 } else {
@@ -84,8 +84,11 @@ exports.pauseDownload = async (download_uid) => {
     } else if (download['finished']) {
         logger.info(`Download ${download_uid} could not be paused before completing.`);
         return false;
+    } else {
+        logger.info(`Pausing download ${download_uid}`);
     }
 
+    killActiveDownload(download);
     return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
 }
 
@@ -120,16 +123,23 @@ exports.cancelDownload = async (download_uid) => {
     } else if (download['finished']) {
         logger.info(`Download ${download_uid} could not be cancelled before completing.`);
         return false;
+    } else {
+        logger.info(`Cancelling download ${download_uid}`);
     }
-    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
+
+    killActiveDownload(download);
+    await handleDownloadError(download_uid, 'Cancelled', 'cancelled');
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true});
 }
 
 exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download, error_message, error_type = null) {
+async function handleDownloadError(download_uid, error_message, error_type = null) {
-    if (!download || !download['uid']) return;
+    if (!download_uid) return;
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['error']) return;
     notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
     await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
@@ -180,7 +190,7 @@ async function checkDownloads() {
             if (waiting_download['sub_id']) {
                 const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
                 if (sub_missing) {
-                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    handleDownloadError(waiting_download['uid'], `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
                     continue;
                 }
             }
@@ -195,6 +205,14 @@ async function checkDownloads() {
     }
 }
 
+function killActiveDownload(download) {
+    const child_process = download_to_child_process[download['uid']];
+    if (download['step_index'] === 2 && child_process) {
+        youtubedl_api.killYoutubeDLProcess(child_process);
+        delete download_to_child_process[download['uid']];
+    }
+}
+
 exports.collectInfo = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     if (download['paused']) {
@@ -232,8 +250,7 @@ exports.collectInfo = async (download_uid) => {
             const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
             logger.warn(error);
             if (download_uid) {
-                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download_uid, error, 'exists_in_archive');
-                await handleDownloadError(download, error, 'exists_in_archive');
                 return;
             }
         }
@@ -276,7 +293,7 @@ exports.collectInfo = async (download_uid) => {
                                                                 });
 }
 
-exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => {
+exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     if (download['paused']) {
         return;
@@ -306,21 +323,25 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
         const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
         const file_objs = [];
         // download file
-        const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
+        let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
+        if (child_process) download_to_child_process[download['uid']] = child_process;
+        const {parsed_output, err} = await callback;
         clearInterval(download_checker);
         let end_time = Date.now();
         let difference = (end_time - start_time)/1000;
         logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
         if (!parsed_output) {
-            logger.error(err.stderr);
+            const errored_download = await db_api.getRecord('download_queue', {uid: download_uid});
-            await handleDownloadError(download, err.stderr, 'unknown_error');
+            if (errored_download['paused']) return;
+            logger.error(err.toString());
+            await handleDownloadError(download_uid, err.toString(), 'unknown_error');
             resolve(false);
             return;
         } else if (parsed_output) {
             if (parsed_output.length === 0 || parsed_output[0].length === 0) {
                 // ERROR!
                 const error_message = `No output received for video download, check if it exists in your archive.`;
-                await handleDownloadError(download, error_message, 'no_output');
+                await handleDownloadError(download_uid, error_message, 'no_output');
                 logger.warn(error_message);
                 resolve(false);
                 return;
@@ -392,7 +413,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
             } else {
                 const error_message = 'Downloaded file failed to result in metadata object.';
                 logger.error(error_message);
-                await handleDownloadError(download, error_message, 'no_metadata');
+                await handleDownloadError(download_uid, error_message, 'no_metadata');
             }
 
             const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -547,14 +568,13 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
 
     new_args.push('--dump-json');
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, new_args);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args);
     if (!parsed_output || parsed_output.length === 0) {
         let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
         if (err.stderr) error_message += `\n\n${err.stderr}`;
         logger.error(error_message);
         if (download_uid) {
-            const download = await db_api.getRecord('download_queue', {uid: download_uid});
+            await handleDownloadError(download_uid, error_message, 'info_retrieve_failed');
-            await handleDownloadError(download, error_message, 'info_retrieve_failed');
         }
         return null;
     }

backend/package.json

@@ -61,6 +61,7 @@
     "read-last-lines": "^1.7.2",
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
+    "tree-kill": "^1.2.2",
     "unzipper": "^0.10.10",
     "uuidv4": "^6.2.13",
     "winston": "^3.7.2",

backend/subscriptions.js

@@ -64,7 +64,7 @@ async function getSubscriptionInfo(sub) {
         }
     }
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
     if (err) {
         logger.error(err.stderr);
         return false;
@@ -226,7 +226,7 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
     // get videos
     logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
     updateSubscriptionProperty(sub, {downloading: false}, user_uid);
     if (!parsed_output) {
         logger.error('Subscription check failed!');
@@ -482,7 +482,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
             const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
             if (output[metric_to_compare] > file_obj[metric_to_compare]) {
                 // download new video as the simulated one is better
-                const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+                const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
                 if (err) {
                     logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                 } else if (parsed_output) {

backend/test/tests.js

@@ -650,6 +650,16 @@ describe('youtube-dl', async function() {
         }
         config_api.setConfigItem('ytdl_default_downloader', original_fork);
     });
+
+    it('Run process', async function() {
+        this.timeout(300000);
+        const downloader_api = require('../downloader');
+        const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
+        const args = await downloader_api.generateArgs(url, 'video', {}, null, true);
+        const {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args);
+        assert(child_process);
+        console.log(await callback);
+    });
 });
 
 describe('Tasks', function() {

backend/youtube-dl.js

@@ -1,5 +1,7 @@
 const fs = require('fs-extra');
 const fetch = require('node-fetch');
+const execa = require('execa');
+const kill = require('tree-kill');
 
 const logger = require('./logger');
 const utils = require('./utils');
@@ -24,7 +26,20 @@ exports.youtubedl_forks = {
     }
 }
 
-exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
+exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
+    let callback = null;
+    let child_process = null;
+    if (downloadMethod) {
+        callback = exports.runYoutubeDLMain(url, args, downloadMethod);
+    } else {
+        ({callback, child_process} = await runYoutubeDLProcess(url, args));
+    }
+
+    return {child_process, callback};
+}
+
+// Run youtube-dl in a main thread (with possible downloadMethod)
+exports.runYoutubeDLMain = async (url, args, downloadMethod = youtubedl.exec) => {
     return new Promise(resolve => {
         downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
             const parsed_output = utils.parseOutputJSON(output, err);
@@ -33,6 +48,30 @@ exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
     });
 }
 
+// Run youtube-dl in a subprocess
+const runYoutubeDLProcess = async (url, args) => {
+    const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity});
+    const callback = new Promise(async resolve => {
+        try {
+            const {stdout, stderr} = await child_process;
+            const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr);
+            resolve({parsed_output, err: stderr});
+        } catch (e) {
+            resolve({parsed_output: null, err: e})
+        }
+    });
+    return {child_process, callback}
+}
+
+async function getYoutubeDLPath() {
+    const guessed_base_path = 'node_modules/youtube-dl/bin/';
+    return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
+}
+
+exports.killYoutubeDLProcess = async (child_process) => {
+    kill(child_process.pid, 'SIGKILL');
+}
+
 exports.checkForYoutubeDLUpdate = async () => {
     const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
     // get current version

src/api-types/models/Download.ts

@@ -8,6 +8,7 @@ export type Download = {
     running: boolean;
     finished: boolean;
     paused: boolean;
+    cancelled?: boolean;
     finished_step: boolean;
     url: string;
     type: string;

src/app/components/downloads/downloads.component.ts

@@ -69,8 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
       tooltip: $localize`Pause`,
       action: (download: Download) => this.pauseDownload(download),
       show: (download: Download) => !download.finished && (!download.paused || !download.finished_step),
-      icon: 'pause',
+      icon: 'pause'
-      loading: (download: Download) => download.paused && !download.finished_step
     },
     {
       tooltip: $localize`Resume`,
@@ -81,7 +80,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     {
       tooltip: $localize`Cancel`,
       action: (download: Download) => this.cancelDownload(download),
-      show: (download: Download) => false && !download.finished && !download.paused, // TODO: add possibility to cancel download
+      show: (download: Download) => !download.finished && !download.paused && !download.cancelled,
       icon: 'cancel'
     },
     {

src/app/main/main.component.ts

@@ -169,6 +169,8 @@ export class MainComponent implements OnInit {
   argsChangedSubject: Subject<boolean> = new Subject<boolean>();
   simulatedOutput = '';
 
+  interval_id = null;
+
   constructor(public postsService: PostsService, private youtubeSearch: YoutubeSearchService, public snackBar: MatSnackBar,
     private router: Router, public dialog: MatDialog, private platform: Platform, private route: ActivatedRoute) {
     this.audioOnly = false;
@@ -232,11 +234,12 @@ export class MainComponent implements OnInit {
     }
 
     // get downloads routine
-    setInterval(() => {
+    if (this.interval_id) { clearInterval(this.interval_id) }
+    this.interval_id = setInterval(() => {
       if (this.current_download) {
         this.getCurrentDownload();
       }
-    }, 500);
+    }, 1000);
 
     return true;
   }
@@ -294,6 +297,10 @@ export class MainComponent implements OnInit {
     }
   }
 
+  ngOnDestroy(): void {
+    if (this.interval_id) { clearInterval(this.interval_id) }
+  }
+
   // download helpers
   downloadHelper(container: DatabaseFile | Playlist, type: string, is_playlist = false, force_view = false, navigate_mode = false): void {
     this.downloadingfile = false;