From 9206d4ba288962842490577596286af74381a259 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 25 Nov 2023 00:40:44 -0500 Subject: [PATCH] Simplified youtube-dl run functions --- backend/downloader.js | 7 +++-- backend/subscriptions.js | 11 ++++--- backend/test/tests.js | 63 ++++++++++++++++++++++++++++++++++++++++ backend/youtube-dl.js | 52 ++++++++++++++++++--------------- 4 files changed, 102 insertions(+), 31 deletions(-) diff --git a/backend/downloader.js b/backend/downloader.js index aa8b13f2..4ef9987d 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -293,7 +293,7 @@ exports.collectInfo = async (download_uid) => { }); } -exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => { +exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { return; @@ -323,7 +323,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => { const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000); const file_objs = []; // download file - let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod); + let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, customDownloadHandler); if (child_process) download_to_child_process[download['uid']] = child_process; const {parsed_output, err} = await callback; clearInterval(download_checker); @@ -568,7 +568,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { new_args.push('--dump-json'); - const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args); + let {callback} = await youtubedl_api.runYoutubeDL(url, new_args); + const {parsed_output, err} = await callback; if (!parsed_output || parsed_output.length === 0) { let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; if (err.stderr) error_message += `\n\n${err.stderr}`; diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 58211b40..fd7de693 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -63,7 +63,8 @@ async function getSubscriptionInfo(sub) { } } - const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); + let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await callback; if (err) { logger.error(err.stderr); return false; @@ -225,8 +226,9 @@ exports.getVideosForSub = async (sub, user_uid = null) => { // get videos logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`); - const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); - updateSubscriptionProperty(sub, {downloading: false}, user_uid); + let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await callback; + updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid); if (!parsed_output) { logger.error('Subscription check failed!'); if (err) logger.error(err); @@ -480,7 +482,8 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; if (info[metric_to_compare] > file_obj[metric_to_compare]) { // download new video as the simulated one is better - const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig); + let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await callback; if (err) { logger.verbose(`Failed to download better version of video ${file_obj['id']}`); } else if (parsed_output) { diff --git a/backend/test/tests.js b/backend/test/tests.js index be61d553..e12720e5 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -662,6 +662,69 @@ describe('youtube-dl', async function() { }); }); +describe('Subscriptions', function() { + const new_sub = { + name: 'test_sub', + url: 'https://www.youtube.com/channel/UCzofo-P8yMMCOv8rsPfIR-g', + maxQuality: null, + id: uuid(), + user_uid: null, + type: 'video', + paused: true + }; + beforeEach(async function() { + await db_api.removeAllRecords('subscriptions'); + }); + it('Subscribe', async function () { + const success = await subscriptions_api.subscribe(new_sub, null, true); + assert(success); + const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(sub_exists); + }); + it('Unsubscribe', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + await subscriptions_api.unsubscribe(new_sub); + const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(!sub_exists); + }); + it('Delete subscription file', async function () { + + }); + it('Get subscription by name', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_by_name = await subscriptions_api.getSubscriptionByName('test_sub'); + assert(sub_by_name); + }); + it('Get subscriptions', async function() { + await subscriptions_api.subscribe(new_sub, null, true); + const subs = await subscriptions_api.getSubscriptions(null); + assert(subs && subs.length === 1); + }); + it('Update subscription', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_update = Object.assign({}, new_sub, {name: 'updated_name'}); + await subscriptions_api.updateSubscription(sub_update); + const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(updated_sub['name'] === 'updated_name'); + }); + it('Update subscription property', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_update = Object.assign({}, new_sub, {name: 'updated_name'}); + await subscriptions_api.updateSubscriptionPropertyMultiple([sub_update], {name: 'updated_name'}); + const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(updated_sub['name'] === 'updated_name'); + }); + it('Write subscription metadata', async function() { + const metadata_path = path.join('subscriptions', 'channels', 'test_sub', 'subscription_backup.json'); + if (fs.existsSync(metadata_path)) fs.unlinkSync(metadata_path); + await subscriptions_api.subscribe(new_sub, null, true); + assert(fs.existsSync(metadata_path)); + }); + it('Fresh uploads', async function() { + + }); +}); + describe('Tasks', function() { const tasks_api = require('../tasks'); beforeEach(async function() { diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js index 5d55688c..a2a6ffb4 100644 --- a/backend/youtube-dl.js +++ b/backend/youtube-dl.js @@ -1,5 +1,6 @@ const fs = require('fs-extra'); const fetch = require('node-fetch'); +const path = require('path'); const execa = require('execa'); const kill = require('tree-kill'); @@ -25,11 +26,11 @@ exports.youtubedl_forks = { } } -exports.runYoutubeDL = async (url, args, downloadMethod = null) => { +exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => { let callback = null; let child_process = null; - if (downloadMethod) { - callback = exports.runYoutubeDLMain(url, args, downloadMethod); + if (customDownloadHandler) { + callback = runYoutubeDLCustom(url, args, customDownloadHandler); } else { ({callback, child_process} = await runYoutubeDLProcess(url, args)); } @@ -37,19 +38,27 @@ exports.runYoutubeDL = async (url, args, downloadMethod = null) => { return {child_process, callback}; } -// Run youtube-dl in a main thread (with possible downloadMethod) -exports.runYoutubeDLMain = async (url, args, downloadMethod = defaultDownloadMethod) => { +// Run youtube-dl directly (not cancellable) +const runYoutubeDLCustom = async (url, args, customDownloadHandler) => { + const downloadHandler = customDownloadHandler; return new Promise(resolve => { - downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) { + downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) { const parsed_output = utils.parseOutputJSON(output, err); resolve({parsed_output, err}); }); }); } -// Run youtube-dl in a subprocess -const runYoutubeDLProcess = async (url, args) => { - const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity}); +// Run youtube-dl in a subprocess (cancellable) +const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => { + const youtubedl_path = getYoutubeDLPath(youtubedl_fork); + const binary_exists = fs.existsSync(youtubedl_path); + if (!binary_exists) { + const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`; + logger.error(err); + return; + } + const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity}); const callback = new Promise(async resolve => { try { const {stdout, stderr} = await child_process; @@ -62,15 +71,10 @@ const runYoutubeDLProcess = async (url, args) => { return {child_process, callback} } -async function getYoutubeDLPath() { - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); -} - -async function defaultDownloadMethod(url, args, options, callback) { - const {child_process, _} = await runYoutubeDLProcess(url, args); - const {stdout, stderr} = await child_process; - return await callback(stderr, stdout.trim().split(/\r?\n/)); +function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) { + const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : ''); + const binary_path = path.join('appdata', 'bin', binary_file_name); + return binary_path; } exports.killYoutubeDLProcess = async (child_process) => { @@ -91,14 +95,13 @@ exports.checkForYoutubeDLUpdate = async () => { let stored_binary_path = current_app_details['path']; if (!stored_binary_path || typeof stored_binary_path !== 'string') { // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); + const guessed_file_path = getYoutubeDLPath(); if (fs.existsSync(guessed_file_path)) { stored_binary_path = guessed_file_path; // logger.info('INFO: Guess successful! Update process continuing...') } else { - logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); - return null; + logger.warn(`youtuble-dl binary not found at '${guessed_file_path}', downloading...`); + return true; } } @@ -107,6 +110,7 @@ exports.checkForYoutubeDLUpdate = async () => { } exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => { + await fs.ensureDir(path.join('appdata', 'bin')); const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path); } @@ -114,7 +118,7 @@ exports.updateYoutubeDL = async (latest_update_version, custom_output_path = nul exports.verifyBinaryExists = () => { const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) { - details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl'; + details_json['path'] = getYoutubeDLPath(); details_json['exec'] = 'youtube-dl'; details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION; fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); @@ -128,7 +132,7 @@ async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, // build the URL const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`; - const output_path = custom_output_path || `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork); await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);