diff --git a/backend/app.js b/backend/app.js index 1077665..f3c08d5 100644 --- a/backend/app.js +++ b/backend/app.js @@ -657,27 +657,13 @@ function generateEnvVarConfigItem(key) { // currently only works for single urls async function getUrlInfos(url) { - let startDate = Date.now(); - let result = []; - return new Promise(resolve => { - youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { - let new_date = Date.now(); - let difference = (new_date - startDate)/1000; - logger.debug(`URL info retrieval delay: ${difference} seconds.`); - if (err) { - logger.error(`Error during retrieving formats for ${url}: ${err}`); - resolve(null); - } - let try_putput = null; - try { - try_putput = JSON.parse(output); - result = try_putput; - } catch(e) { - logger.error(`Failed to retrieve available formats for url: ${url}`); - } - resolve(result); - }); - }); + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, ['--dump-json']); + if (!parsed_output || parsed_output.length !== 1) { + logger.error(`Failed to retrieve available formats for url: ${url}`); + if (err) logger.error(err); + return null; + } + return parsed_output[0]; } // youtube-dl functions diff --git a/backend/downloader.js b/backend/downloader.js index d37a275..a7adc62 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -187,7 +187,7 @@ async function checkDownloads() { // move to next step running_downloads_count++; if (waiting_download['step_index'] === 0) { - collectInfo(waiting_download['uid']); + exports.collectInfo(waiting_download['uid']); } else if (waiting_download['step_index'] === 1) { exports.downloadQueuedFile(waiting_download['uid']); } @@ -195,7 +195,7 @@ async function checkDownloads() { } } -async function collectInfo(download_uid) { +exports.collectInfo = async (download_uid) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { return; @@ -218,17 +218,18 @@ async function collectInfo(download_uid) { // get video info prior to download let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid); - if (!info) { + if (!info || info.length === 0) { // info failed, error presumably already recorded return; } // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive && !options.ignoreArchive) { - const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']); + if (useYoutubeDLArchive && !options.ignoreArchive && info.length === 1) { + const info_obj = info[0]; + const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info_obj['id'], type, download['user_uid'], download['sub_id']); if (exists_in_archive) { - const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`; + const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`; logger.warn(error); if (download_uid) { const download = await db_api.getRecord('download_queue', {uid: download_uid}); @@ -241,7 +242,7 @@ async function collectInfo(download_uid) { let category = null; // check if it fits into a category. If so, then get info again using new args - if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); + if (info.length === 1 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); // set custom output if the category has one and re-retrieve info so the download manager has the right file name if (category && category['custom_output']) { @@ -262,14 +263,14 @@ async function collectInfo(download_uid) { // store info in download for future use for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename'])); - const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null; + const title = info.length > 1 ? info[0]['playlist_title'] || info[0]['playlist'] : info[0]['title']; await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args, finished_step: true, running: false, options: options, files_to_check_for_progress: files_to_check_for_progress, expected_file_size: expected_file_size, - title: playlist_title ? playlist_title : info['title'], + title: title, category: stripped_category, prefetched_info: null }); @@ -385,8 +386,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec if (file_objs.length > 1) { // create playlist - const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); - container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']); + container = await files_api.createPlaylist(download['title'], file_objs.map(file_obj => file_obj.uid), download['user_uid']); } else if (file_objs.length === 1) { container = file_objs[0]; } else { @@ -536,34 +536,30 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f } exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { - return new Promise(resolve => { - // remove bad args - const temp_args = utils.filterArgs(args, ['--no-simulate']); - const new_args = [...temp_args]; + // remove bad args + const temp_args = utils.filterArgs(args, ['--no-simulate']); + const new_args = [...temp_args]; - const archiveArgIndex = new_args.indexOf('--download-archive'); - if (archiveArgIndex !== -1) { - new_args.splice(archiveArgIndex, 2); + const archiveArgIndex = new_args.indexOf('--download-archive'); + if (archiveArgIndex !== -1) { + new_args.splice(archiveArgIndex, 2); + } + + new_args.push('--dump-json'); + + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, new_args); + if (!parsed_output || parsed_output.length === 0) { + let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; + if (err.stderr) error_message += `\n\n${err.stderr}`; + logger.error(error_message); + if (download_uid) { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + await handleDownloadError(download, error_message, 'info_retrieve_failed'); } + return null; + } - new_args.push('--dump-json'); - - youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => { - const parsed_output = utils.parseOutputJSON(output, err); - if (parsed_output) { - resolve(parsed_output); - } else { - let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; - if (err.stderr) error_message += `\n\n${err.stderr}`; - logger.error(error_message); - if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error_message, 'info_retrieve_failed'); - } - resolve(null); - } - }); - }); + return parsed_output; } function filterArgs(args, isAudio) { diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 547ac80..4d2c9da 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -2,6 +2,7 @@ const fs = require('fs-extra'); const path = require('path'); const youtubedl = require('youtube-dl'); +const youtubedl_api = require('./youtube-dl'); const config_api = require('./config'); const archive_api = require('./archive'); const utils = require('./utils'); @@ -63,52 +64,36 @@ async function getSubscriptionInfo(sub) { } } - return new Promise(async resolve => { - youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => { - if (debugMode) { - logger.info('Subscribe: got info for subscription ' + sub.id); - } - if (err) { - logger.error(err.stderr); - resolve(false); - } else if (output) { - if (output.length === 0 || (output.length === 1 && output[0] === '')) { - logger.verbose('Could not get info for ' + sub.id); - resolve(false); - } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - if (!sub.name) { - if (sub.isPlaylist) { - sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist; - } else { - sub.name = output_json.uploader; - } - // if it's now valid, update - if (sub.name) { - let sub_name = sub.name; - const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid}); - if (sub_name_exists) sub_name += ` - ${sub.id}`; - await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name}); - } - } + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + if (err) { + logger.error(err.stderr); + return false; + } + logger.verbose('Subscribe: got info for subscription ' + sub.id); + for (const output_json of parsed_output) { + if (!output_json) { + continue; + } - // TODO: get even more info - - resolve(true); - } - resolve(false); + if (!sub.name) { + if (sub.isPlaylist) { + sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist; + } else { + sub.name = output_json.uploader; } - }); - }); + // if it's now valid, update + if (sub.name) { + let sub_name = sub.name; + const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid}); + if (sub_name_exists) sub_name += ` - ${sub.id}`; + await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name}); + } + } + + return true; + } + + return false; } exports.unsubscribe = async (sub, deleteMode, user_uid = null) => { @@ -241,33 +226,24 @@ exports.getVideosForSub = async (sub, user_uid = null) => { // get videos logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`); - return new Promise(async resolve => { - youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { - // cleanup - updateSubscriptionProperty(sub, {downloading: false}, user_uid); + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + updateSubscriptionProperty(sub, {downloading: false}, user_uid); + if (!parsed_output) { + logger.error('Subscription check failed!'); + if (err) logger.error(err); + return null; + } - // remove temporary archive file if it exists - const archive_path = path.join(appendedBasePath, 'archive.txt'); - const archive_exists = await fs.pathExists(archive_path); - if (archive_exists) { - await fs.unlink(archive_path); - } + // remove temporary archive file if it exists + const archive_path = path.join(appendedBasePath, 'archive.txt'); + const archive_exists = await fs.pathExists(archive_path); + if (archive_exists) { + await fs.unlink(archive_path); + } - logger.verbose('Subscription: finished check for ' + sub.name); - const parsed_output = utils.parseOutputJSON(output, err); - if (!parsed_output) { - logger.error('Subscription check failed!'); - resolve(null); - return; - } - const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid); - resolve(files_to_download); - return; - }); - }, err => { - logger.error(err); - updateSubscriptionProperty(sub, {downloading: false}, user_uid); - }); + logger.verbose('Subscription: finished check for ' + sub.name); + const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid); + return files_to_download; } async function handleOutputJSON(output_jsons, sub, user_uid) { @@ -506,14 +482,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; if (output[metric_to_compare] > file_obj[metric_to_compare]) { // download new video as the simulated one is better - youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => { - if (err) { - logger.verbose(`Failed to download better version of video ${file_obj['id']}`); - } else if (output) { - logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); - await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}); - } - }); + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + if (err) { + logger.verbose(`Failed to download better version of video ${file_obj['id']}`); + } else if (parsed_output) { + logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); + await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}); + } } } });