Subscription videos now skip the collect info step during download process, reducing calls to video host

Public API v1.yaml

@@ -2512,6 +2512,8 @@ components:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key

backend/downloader.js

@@ -26,7 +26,7 @@ if (db_api.database_initialized) {
     });
 }
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -35,6 +35,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             user_uid: user_uid,
             sub_id: sub_id,
             sub_name: sub_name,
+            prefetched_info: prefetched_info,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -185,7 +186,7 @@ async function collectInfo(download_uid) {
     let args = await exports.generateArgs(url, type, options, download['user_uid']);
 
     // get video info prior to download
-    let info = await exports.getVideoInfoByURL(url, args, download_uid);
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
     if (!info) {
         // info failed, error presumably already recorded
@@ -227,7 +228,8 @@ async function collectInfo(download_uid) {
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    prefetched_info: null
                                                                 });
 }
 
@@ -572,8 +574,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
 function filterArgs(args, isAudio) {
     const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
     const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
-    return args.filter(x => !args_to_remove.includes(x));
 }
 
 async function checkDownloadPercent(download_uid) {

backend/subscriptions.js

@@ -296,7 +296,8 @@ async function getVideosForSub(sub, user_uid = null) {
 
                 for (let j = 0; j < files_to_download.length; j++) {
                     const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
+                    file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
                 }
 
                 resolve(files_to_download);
@@ -420,6 +421,8 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push('--no-clean-infojson');
     }
 
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
     return downloadConfig;
 }
 

backend/test/tests.js

@@ -618,4 +618,12 @@ describe('Archive', async function() {
         assert(!new_archive.includes('testing2'));
         assert(new_blacklist.includes('testing2'));
     });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
+    });
 });

backend/utils.js

@@ -481,6 +481,10 @@ function injectArgs(original_args, new_args) {
     return updated_args;
 }
 
+function filterArgs(args, args_to_remove) {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
 const searchObjectByString = function(o, s) {
     s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
     s = s.replace(/^\./, '');           // strip a leading dot
@@ -496,6 +500,22 @@ const searchObjectByString = function(o, s) {
     return o;
 }
 
+function stripPropertiesFromObject(obj, properties, whitelist = false) {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
 function getArchiveFolder(type, user_uid = null, sub = null) {
     const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
     const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
@@ -561,7 +581,9 @@ module.exports = {
     fetchFile: fetchFile,
     restartServer: restartServer,
     injectArgs: injectArgs,
+    filterArgs: filterArgs,
     searchObjectByString: searchObjectByString,
+    stripPropertiesFromObject: stripPropertiesFromObject,
     getArchiveFolder: getArchiveFolder,
     File: File
 }

src/api-types/models/Download.ts

@@ -22,4 +22,5 @@ export type Download = {
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
+    prefetched_info?: any;
 };