Simplified youtube-dl run functions

backend/downloader.js

@@ -293,7 +293,7 @@ exports.collectInfo = async (download_uid) => {
                                                                 });
 }
 
-exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
+exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     if (download['paused']) {
         return;
@@ -323,7 +323,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
         const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
         const file_objs = [];
         // download file
-        let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
+        let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, customDownloadHandler);
         if (child_process) download_to_child_process[download['uid']] = child_process;
         const {parsed_output, err} = await callback;
         clearInterval(download_checker);
@@ -568,7 +568,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
 
     new_args.push('--dump-json');
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args);
+    let {callback} = await youtubedl_api.runYoutubeDL(url, new_args);
+    const {parsed_output, err} = await callback;
     if (!parsed_output || parsed_output.length === 0) {
         let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
         if (err.stderr) error_message += `\n\n${err.stderr}`;

backend/subscriptions.js

@@ -63,7 +63,8 @@ async function getSubscriptionInfo(sub) {
         }
     }
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
+    let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+    const {parsed_output, err} = await callback;
     if (err) {
         logger.error(err.stderr);
         return false;
@@ -225,8 +226,9 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
     // get videos
     logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
+    let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
-    updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+    const {parsed_output, err} = await callback;
+    updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid);
     if (!parsed_output) {
         logger.error('Subscription check failed!');
         if (err) logger.error(err);
@@ -480,7 +482,8 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
         const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
         if (info[metric_to_compare] > file_obj[metric_to_compare]) {
             // download new video as the simulated one is better
-            const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
+            let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
+            const {parsed_output, err} = await callback;
             if (err) {
                 logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
             } else if (parsed_output) {

backend/test/tests.js

@@ -662,6 +662,69 @@ describe('youtube-dl', async function() {
     });
 });
 
+describe('Subscriptions', function() {
+    const new_sub = {
+        name: 'test_sub',
+        url: 'https://www.youtube.com/channel/UCzofo-P8yMMCOv8rsPfIR-g',
+        maxQuality: null,
+        id: uuid(),
+        user_uid: null,
+        type: 'video',
+        paused: true
+    };
+    beforeEach(async function() {
+        await db_api.removeAllRecords('subscriptions');
+    });
+    it('Subscribe', async function () {
+        const success = await subscriptions_api.subscribe(new_sub, null, true);
+        assert(success);
+        const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
+        assert(sub_exists);
+    });
+    it('Unsubscribe', async function () {
+        await subscriptions_api.subscribe(new_sub, null, true);
+        await subscriptions_api.unsubscribe(new_sub);
+        const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
+        assert(!sub_exists);
+    });
+    it('Delete subscription file', async function () {
+        
+    });
+    it('Get subscription by name', async function () {
+        await subscriptions_api.subscribe(new_sub, null, true);
+        const sub_by_name = await subscriptions_api.getSubscriptionByName('test_sub');
+        assert(sub_by_name);
+    });
+    it('Get subscriptions', async function() {
+        await subscriptions_api.subscribe(new_sub, null, true);
+        const subs = await subscriptions_api.getSubscriptions(null);
+        assert(subs && subs.length === 1);
+    });
+    it('Update subscription', async function () {
+        await subscriptions_api.subscribe(new_sub, null, true);
+        const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
+        await subscriptions_api.updateSubscription(sub_update);
+        const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
+        assert(updated_sub['name'] === 'updated_name');
+    });
+    it('Update subscription property', async function () {
+        await subscriptions_api.subscribe(new_sub, null, true);
+        const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
+        await subscriptions_api.updateSubscriptionPropertyMultiple([sub_update], {name: 'updated_name'});
+        const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
+        assert(updated_sub['name'] === 'updated_name');
+    });
+    it('Write subscription metadata', async function() {
+        const metadata_path = path.join('subscriptions', 'channels', 'test_sub', 'subscription_backup.json');
+        if (fs.existsSync(metadata_path)) fs.unlinkSync(metadata_path);
+        await subscriptions_api.subscribe(new_sub, null, true);
+        assert(fs.existsSync(metadata_path));
+    });
+    it('Fresh uploads', async function() {
+
+    });
+});
+
 describe('Tasks', function() {
     const tasks_api = require('../tasks');
     beforeEach(async function() {

backend/youtube-dl.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const fetch = require('node-fetch');
+const path = require('path');
 const execa = require('execa');
 const kill = require('tree-kill');
 
@@ -25,11 +26,11 @@ exports.youtubedl_forks = {
     }
 }
 
-exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
+exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => {
     let callback = null;
     let child_process = null;
-    if (downloadMethod) {
+    if (customDownloadHandler) {
-        callback = exports.runYoutubeDLMain(url, args, downloadMethod);
+        callback = runYoutubeDLCustom(url, args, customDownloadHandler);
     } else {
         ({callback, child_process} = await runYoutubeDLProcess(url, args));
     }
@@ -37,19 +38,27 @@ exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
     return {child_process, callback};
 }
 
-// Run youtube-dl in a main thread (with possible downloadMethod)
+// Run youtube-dl directly (not cancellable)
-exports.runYoutubeDLMain = async (url, args, downloadMethod = defaultDownloadMethod) => {
+const runYoutubeDLCustom = async (url, args, customDownloadHandler) => {
+    const downloadHandler = customDownloadHandler;
     return new Promise(resolve => {
-        downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
+        downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) {
             const parsed_output = utils.parseOutputJSON(output, err);
             resolve({parsed_output, err});
         });
     });
 }
 
-// Run youtube-dl in a subprocess
+// Run youtube-dl in a subprocess (cancellable)
-const runYoutubeDLProcess = async (url, args) => {
+const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => {
-    const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity});
+    const youtubedl_path = getYoutubeDLPath(youtubedl_fork);
+    const binary_exists = fs.existsSync(youtubedl_path);
+    if (!binary_exists) {
+        const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`;
+        logger.error(err);
+        return;
+    }
+    const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity});
     const callback = new Promise(async resolve => {
         try {
             const {stdout, stderr} = await child_process;
@@ -62,15 +71,10 @@ const runYoutubeDLProcess = async (url, args) => {
     return {child_process, callback}
 }
 
-async function getYoutubeDLPath() {
+function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) {
-    const guessed_base_path = 'node_modules/youtube-dl/bin/';
+    const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : '');
-    return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
+    const binary_path = path.join('appdata', 'bin', binary_file_name);
-}
+    return binary_path;
-
-async function defaultDownloadMethod(url, args, options, callback) {
-    const {child_process, _} = await runYoutubeDLProcess(url, args);
-    const {stdout, stderr} = await child_process;
-    return await callback(stderr, stdout.trim().split(/\r?\n/));
 }
 
 exports.killYoutubeDLProcess = async (child_process) => {
@@ -91,14 +95,13 @@ exports.checkForYoutubeDLUpdate = async () => {
     let stored_binary_path = current_app_details['path'];
     if (!stored_binary_path || typeof stored_binary_path !== 'string') {
         // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
-        const guessed_base_path = 'node_modules/youtube-dl/bin/';
+        const guessed_file_path = getYoutubeDLPath();
-        const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
         if (fs.existsSync(guessed_file_path)) {
             stored_binary_path = guessed_file_path;
             // logger.info('INFO: Guess successful! Update process continuing...')
         } else {
-            logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
+            logger.warn(`youtuble-dl binary not found at '${guessed_file_path}', downloading...`);
-            return null;
+            return true;
         }
     }
 
@@ -107,6 +110,7 @@ exports.checkForYoutubeDLUpdate = async () => {
 }
 
 exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => {
+    await fs.ensureDir(path.join('appdata', 'bin'));
     const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
     await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path);
 }
@@ -114,7 +118,7 @@ exports.updateYoutubeDL = async (latest_update_version, custom_output_path = nul
 exports.verifyBinaryExists = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
-        details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
+        details_json['path'] = getYoutubeDLPath();
         details_json['exec'] = 'youtube-dl';
         details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION;
         fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
@@ -128,7 +132,7 @@ async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version,
 
     // build the URL
     const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
-    const output_path = custom_output_path || `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+    const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork);
 
     await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);