diff --git a/Public API v1.yaml b/Public API v1.yaml index 37b9f1f..1d9e0dd 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -2512,6 +2512,8 @@ components: type: string sub_name: type: string + prefetched_info: + type: object Task: required: - key diff --git a/backend/downloader.js b/backend/downloader.js index 1ac0260..a0ae6e6 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -26,7 +26,7 @@ if (db_api.database_initialized) { }); } -exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => { +exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => { return await mutex.runExclusive(async () => { const download = { url: url, @@ -35,6 +35,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu user_uid: user_uid, sub_id: sub_id, sub_name: sub_name, + prefetched_info: prefetched_info, options: options, uid: uuid(), step_index: 0, @@ -185,7 +186,7 @@ async function collectInfo(download_uid) { let args = await exports.generateArgs(url, type, options, download['user_uid']); // get video info prior to download - let info = await exports.getVideoInfoByURL(url, args, download_uid); + let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid); if (!info) { // info failed, error presumably already recorded @@ -227,7 +228,8 @@ async function collectInfo(download_uid) { options: options, files_to_check_for_progress: files_to_check_for_progress, expected_file_size: expected_file_size, - title: playlist_title ? playlist_title : info['title'] + title: playlist_title ? playlist_title : info['title'], + prefetched_info: null }); } @@ -572,8 +574,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { function filterArgs(args, isAudio) { const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs']; const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail']; - const args_to_remove = isAudio ? video_only_args : audio_only_args; - return args.filter(x => !args_to_remove.includes(x)); + return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args); } async function checkDownloadPercent(download_uid) { diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 8164eb0..295818b 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -296,7 +296,8 @@ async function getVideosForSub(sub, user_uid = null) { for (let j = 0; j < files_to_download.length; j++) { const file_to_download = files_to_download[j]; - await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name); + file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']); // prevent download object from blowing up in size + await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download); } resolve(files_to_download); @@ -420,6 +421,8 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push('--no-clean-infojson'); } + downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']); + return downloadConfig; } diff --git a/backend/test/tests.js b/backend/test/tests.js index 15856c9..b640687 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -618,4 +618,12 @@ describe('Archive', async function() { assert(!new_archive.includes('testing2')); assert(new_blacklist.includes('testing2')); }); +}); + +describe('Utils', async function() { + it('Strip properties', async function() { + const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'}; + const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']); + assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3']) + }); }); \ No newline at end of file diff --git a/backend/utils.js b/backend/utils.js index 2ccd00b..4c36230 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -481,6 +481,10 @@ function injectArgs(original_args, new_args) { return updated_args; } +function filterArgs(args, args_to_remove) { + return args.filter(x => !args_to_remove.includes(x)); +} + const searchObjectByString = function(o, s) { s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties s = s.replace(/^\./, ''); // strip a leading dot @@ -496,6 +500,22 @@ const searchObjectByString = function(o, s) { return o; } +function stripPropertiesFromObject(obj, properties, whitelist = false) { + if (!whitelist) { + const new_obj = JSON.parse(JSON.stringify(obj)); + for (let field of properties) { + delete new_obj[field]; + } + return new_obj; + } + + const new_obj = {}; + for (let field of properties) { + new_obj[field] = obj[field]; + } + return new_obj; +} + function getArchiveFolder(type, user_uid = null, sub = null) { const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path'); const subsFolderPath = config_api.getConfigItem('ytdl_subscriptions_base_path'); @@ -561,7 +581,9 @@ module.exports = { fetchFile: fetchFile, restartServer: restartServer, injectArgs: injectArgs, + filterArgs: filterArgs, searchObjectByString: searchObjectByString, + stripPropertiesFromObject: stripPropertiesFromObject, getArchiveFolder: getArchiveFolder, File: File } diff --git a/src/api-types/models/Download.ts b/src/api-types/models/Download.ts index 6556da0..db447c3 100644 --- a/src/api-types/models/Download.ts +++ b/src/api-types/models/Download.ts @@ -22,4 +22,5 @@ export type Download = { user_uid?: string; sub_id?: string; sub_name?: string; + prefetched_info?: any; }; \ No newline at end of file