Additional scaffolding for download manager

Added queue to npm backend dependencies

backend/downloader.js

@@ -1,4 +1,42 @@
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+const path = require('path');
+const queue = require('queue');
 
+const youtubedl = require('youtube-dl');
+const config_api = require('./config');
+const twitch_api = require('./twitch');
+const utils = require('./utils');
+
+let db_api = null;
+let logger = null;
+
+function setDB(input_db_api) { db_api = input_db_api }
+function setLogger(input_logger) { logger = input_logger; }
+
+exports.initialize = (input_db_api, input_logger) => {
+    setDB(input_db_api);
+    setLogger(input_logger);
+}
+
+exports.pauseDownload = () => {
+
+}
+
+async function checkDownloads() {
+    const downloads = await db_api.getRecords('download_queue');
+    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
+    downloads = downloads.filter(download => !download.paused);
+    for (let i = 0; i < downloads.length; i++) {
+        if (i === config_api.getConfigItem('ytdl_'))
+    }
+}
+
+async function createDownload(url, type, options) {
+    const download = {url: url, type: type, options: options, uid: uuid()};
+    await db_api.insertRecord(download);
+    return download;
+}
 
 async function collectInfo(download_uid) {
     const download = db_api.getRecord('download_queue', {uid: download_uid});
@@ -32,41 +70,20 @@ async function collectInfo(download_uid) {
     await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info});
 }
 
-async function downloadFileByURL_exec(url, type, options) {
+async function downloadQueuedFile(url, type, options) {
-    const download = db_api.getRecord('download_queue', {uid: download_uid});
-
-    const url = download['url'];
-    const type = download['type'];
-    const options = download['options'];
-    const args = download['args'];
-    const category = download['category'];
-    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
-    if (options.user) {
-        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const user_path = path.join(usersFileFolder, options.user, type);
-        fs.ensureDirSync(user_path);
-        fileFolderPath = user_path + path.sep;
-        multiUserMode = {
-            user: options.user,
-            file_path: fileFolderPath
-        }
-        options.customFileFolderPath = fileFolderPath;
-    }
 
 }
 
-async function downloadFileByURL_exec_old(url, type, options, sessionID = null) {
+async function downloadFileByURL_exec(url, type, options) {
-    return new Promise(async resolve => {
+    return new Promise(resolve => {
-        var date = Date.now();
+        const download = db_api.getRecord('download_queue', {uid: download_uid});
 
-        // audio / video specific vars
+        const url = download['url'];
-        var is_audio = type === 'audio';
+        const type = download['type'];
-        var ext = is_audio ? '.mp3' : '.mp4';
+        const options = download['options'];
-        var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
+        const args = download['args'];
-        let category = null;
+        const category = download['category'];
-
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
-        // prepend with user if needed
-        let multiUserMode = null;
         if (options.user) {
             let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
             const user_path = path.join(usersFileFolder, options.user, type);
@@ -75,89 +92,26 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null)
             options.customFileFolderPath = fileFolderPath;
         }
 
-        options.downloading_method = 'exec';
-        let downloadConfig = await generateArgs(url, type, options);
-
-        // adds download to download helper
-        const download_uid = uuid();
-        const session = sessionID ? sessionID : 'undeclared';
-        let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
-        if (!session_downloads) {
-            session_downloads = {session_id: session};
-            downloads.push(session_downloads);
-        }
-        session_downloads[download_uid] = {
-            uid: download_uid,
-            ui_uid: options.ui_uid,
-            downloading: true,
-            complete: false,
-            url: url,
-            type: type,
-            percent_complete: 0,
-            is_playlist: url.includes('playlist'),
-            timestamp_start: Date.now(),
-            filesize: null
-        };
-        const download = session_downloads[download_uid];
-        updateDownloads();
-
-        let download_checker = null;
-
-        // get video info prior to download
-        let info = await getVideoInfoByURL(url, downloadConfig, download);
-        if (!info && url.includes('youtu')) {
-            resolve(false);
-            return;
-        } else if (info) {
-            // check if it fits into a category. If so, then get info again using new downloadConfig
-            if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
-
-            // set custom output if the category has one and re-retrieve info so the download manager has the right file name
-            if (category && category['custom_output']) {
-                options.customOutput = category['custom_output'];
-                options.noRelativePath = true;
-                downloadConfig = await generateArgs(url, type, options);
-                info = await getVideoInfoByURL(url, downloadConfig, download);
-            }
-
-            // store info in download for future use
-            if (Array.isArray(info)) {
-                download['fileNames'] = [];
-                for (let info_obj of info) download['fileNames'].push(info_obj['_filename']);
-            } else {
-                download['_filename'] = info['_filename'];
-            }
-            download['filesize'] = utils.getExpectedFileSize(info);
-            download_checker = setInterval(() => checkDownloadPercent(download), 1000);
-        }
-
         // download file
-        youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
+        youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
-            if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
+            const file_objs = [];
-
-            download['downloading'] = false;
-            download['timestamp_end'] = Date.now();
-            var file_objs = [];
             let new_date = Date.now();
             let difference = (new_date - date)/1000;
             logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
             if (err) {
                 logger.error(err.stderr);
 
-                download['error'] = err.stderr;
-                updateDownloads();
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
-                    download['error'] = 'No output. Check if video already exists in your archive.';
+                    // ERROR!
                     logger.warn(`No output received for video download, check if it exists in your archive.`)
-                    updateDownloads();
 
                     resolve(false);
                     return;
                 }
-                var file_names = [];
+
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
@@ -201,9 +155,6 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null)
                         if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
                     }
 
-                    const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length);
-                    const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null;
-
                     if (options.cropFileSettings) {
                         await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
                     }
@@ -211,14 +162,9 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null)
                     // registers file in DB
                     const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings);
 
-                    // TODO: remove the following line
-                    if (file_name) file_names.push(file_name);
-
                     file_objs.push(file_obj);
                 }
 
-                let is_playlist = file_names.length > 1;
-
                 if (options.merged_string !== null && options.merged_string !== undefined) {
                     let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
                     let diff = current_merged_archive.replace(options.merged_string, '');
@@ -226,16 +172,11 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null)
                     fs.appendFileSync(archive_path, diff);
                 }
 
-                download['complete'] = true;
-                download['fileNames'] = is_playlist ? file_names : [full_file_path]
-                updateDownloads();
-
                 let container = null;
 
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
                     container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
@@ -254,6 +195,135 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null)
 
 // helper functions
 
+async function generateArgs(url, type, options) {
+    const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+    const globalArgs = config_api.getConfigItem('ytdl_custom_args');
+    const useCookies = config_api.getConfigItem('ytdl_use_cookies');
+    const is_audio = type === 'audio';
+
+    const fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+
+    if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
+
+    const customArgs = options.customArgs;
+    const customOutput = options.customOutput;
+    const customQualityConfiguration = options.customQualityConfiguration;
+
+    // video-specific args
+    const selectedHeight = options.selectedHeight;
+
+    // audio-specific args
+    const maxBitrate = options.maxBitrate;
+
+    const youtubeUsername = options.youtubeUsername;
+    const youtubePassword = options.youtubePassword;
+
+    let downloadConfig = null;
+    let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
+    const is_youtube = url.includes('youtu');
+    if (!is_audio && !is_youtube) {
+        // tiktok videos fail when using the default format
+        qualityPath = null;
+    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
+        qualityPath = ['-f', 'bestvideo+bestaudio']
+    }
+
+    if (customArgs) {
+        downloadConfig = customArgs.split(',,');
+    } else {
+        if (customQualityConfiguration) {
+            qualityPath = ['-f', customQualityConfiguration];
+        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
+            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+        } else if (is_audio) {
+            qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
+        }
+
+        if (customOutput) {
+            customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
+            downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
+        } else {
+            downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
+        }
+
+        if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath);
+
+        if (is_audio && !options.skip_audio_args) {
+            downloadConfig.push('-x');
+            downloadConfig.push('--audio-format', 'mp3');
+        }
+
+        if (youtubeUsername && youtubePassword) {
+            downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
+        }
+
+        if (useCookies) {
+            if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+                downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
+            } else {
+                logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
+            }
+        }
+
+        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
+            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
+        }
+
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath;
+            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+
+            await fs.ensureDir(archive_folder);
+            await fs.ensureFile(archive_path);
+
+            let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
+            await fs.ensureFile(blacklist_path);
+
+            let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
+            await fs.ensureFile(merged_path);
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            let status = await mergeFiles(inputPathList, merged_path);
+
+            options.merged_string = await fs.readFile(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
+        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+            downloadConfig.push('--write-thumbnail');
+        }
+
+        if (globalArgs && globalArgs !== '') {
+            // adds global args
+            if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
+                // if global args has an output, replce the original output with that of global args
+                const original_output_index = downloadConfig.indexOf('-o');
+                downloadConfig.splice(original_output_index, 2);
+            }
+            downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
+        }
+
+        const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
+        if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
+            downloadConfig.push('-r', rate_limit);
+        }
+        
+        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+        if (default_downloader === 'yt-dlp') {
+            downloadConfig.push('--no-clean-infojson');
+        }
+
+    }
+
+    // filter out incompatible args
+    downloadConfig = filterArgs(downloadConfig, is_audio);
+
+    logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    return downloadConfig;
+}
+
 async function getVideoInfoByURL(url, args = [], download = null) {
     return new Promise(resolve => {
         // remove bad args
@@ -281,4 +351,4 @@ async function getVideoInfoByURL(url, args = [], download = null) {
             }
         });
     });
-}
+}

backend/package-lock.json

@@ -2719,6 +2719,14 @@
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
       "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
     },
+    "queue": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz",
+      "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==",
+      "requires": {
+        "inherits": "~2.0.3"
+      }
+    },
     "randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",

backend/package.json

@@ -59,6 +59,7 @@
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
     "ps-node": "^0.1.6",
+    "queue": "^6.0.2",
     "read-last-lines": "^1.7.2",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",