diff --git a/backend/downloader.js b/backend/downloader.js index 4e44e86..5054e06 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -1,4 +1,42 @@ +const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); +const path = require('path'); +const queue = require('queue'); +const youtubedl = require('youtube-dl'); +const config_api = require('./config'); +const twitch_api = require('./twitch'); +const utils = require('./utils'); + +let db_api = null; +let logger = null; + +function setDB(input_db_api) { db_api = input_db_api } +function setLogger(input_logger) { logger = input_logger; } + +exports.initialize = (input_db_api, input_logger) => { + setDB(input_db_api); + setLogger(input_logger); +} + +exports.pauseDownload = () => { + +} + +async function checkDownloads() { + const downloads = await db_api.getRecords('download_queue'); + downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start); + downloads = downloads.filter(download => !download.paused); + for (let i = 0; i < downloads.length; i++) { + if (i === config_api.getConfigItem('ytdl_')) + } +} + +async function createDownload(url, type, options) { + const download = {url: url, type: type, options: options, uid: uuid()}; + await db_api.insertRecord(download); + return download; +} async function collectInfo(download_uid) { const download = db_api.getRecord('download_queue', {uid: download_uid}); @@ -32,41 +70,20 @@ async function collectInfo(download_uid) { await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info}); } -async function downloadFileByURL_exec(url, type, options) { - const download = db_api.getRecord('download_queue', {uid: download_uid}); - - const url = download['url']; - const type = download['type']; - const options = download['options']; - const args = download['args']; - const category = download['category']; - let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } +async function downloadQueuedFile(url, type, options) { } -async function downloadFileByURL_exec_old(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); +async function downloadFileByURL_exec(url, type, options) { + return new Promise(resolve => { + const download = db_api.getRecord('download_queue', {uid: download_uid}); - // audio / video specific vars - var is_audio = type === 'audio'; - var ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; - let category = null; - - // prepend with user if needed - let multiUserMode = null; + const url = download['url']; + const type = download['type']; + const options = download['options']; + const args = download['args']; + const category = download['category']; + let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix if (options.user) { let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const user_path = path.join(usersFileFolder, options.user, type); @@ -75,89 +92,26 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null) options.customFileFolderPath = fileFolderPath; } - options.downloading_method = 'exec'; - let downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session); - if (!session_downloads) { - session_downloads = {session_id: session}; - downloads.push(session_downloads); - } - session_downloads[download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now(), - filesize: null - }; - const download = session_downloads[download_uid]; - updateDownloads(); - - let download_checker = null; - - // get video info prior to download - let info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info && url.includes('youtu')) { - resolve(false); - return; - } else if (info) { - // check if it fits into a category. If so, then get info again using new downloadConfig - if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); - - // set custom output if the category has one and re-retrieve info so the download manager has the right file name - if (category && category['custom_output']) { - options.customOutput = category['custom_output']; - options.noRelativePath = true; - downloadConfig = await generateArgs(url, type, options); - info = await getVideoInfoByURL(url, downloadConfig, download); - } - - // store info in download for future use - if (Array.isArray(info)) { - download['fileNames'] = []; - for (let info_obj of info) download['fileNames'].push(info_obj['_filename']); - } else { - download['_filename'] = info['_filename']; - } - download['filesize'] = utils.getExpectedFileSize(info); - download_checker = setInterval(() => checkDownloadPercent(download), 1000); - } - // download file - youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { - if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) - - download['downloading'] = false; - download['timestamp_end'] = Date.now(); - var file_objs = []; + youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) { + const file_objs = []; let new_date = Date.now(); let difference = (new_date - date)/1000; logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); if (err) { logger.error(err.stderr); - download['error'] = err.stderr; - updateDownloads(); resolve(false); return; } else if (output) { if (output.length === 0 || output[0].length === 0) { - download['error'] = 'No output. Check if video already exists in your archive.'; + // ERROR! logger.warn(`No output received for video download, check if it exists in your archive.`) - updateDownloads(); resolve(false); return; } - var file_names = []; + for (let i = 0; i < output.length; i++) { let output_json = null; try { @@ -201,9 +155,6 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null) if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); } - const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); - const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; - if (options.cropFileSettings) { await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext); } @@ -211,14 +162,9 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null) // registers file in DB const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings); - // TODO: remove the following line - if (file_name) file_names.push(file_name); - file_objs.push(file_obj); } - let is_playlist = file_names.length > 1; - if (options.merged_string !== null && options.merged_string !== undefined) { let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); let diff = current_merged_archive.replace(options.merged_string, ''); @@ -226,16 +172,11 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null) fs.appendFileSync(archive_path, diff); } - download['complete'] = true; - download['fileNames'] = is_playlist ? file_names : [full_file_path] - updateDownloads(); - let container = null; if (file_objs.length > 1) { // create playlist const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); - const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user); } else if (file_objs.length === 1) { container = file_objs[0]; @@ -254,6 +195,135 @@ async function downloadFileByURL_exec_old(url, type, options, sessionID = null) // helper functions +async function generateArgs(url, type, options) { + const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; + const globalArgs = config_api.getConfigItem('ytdl_custom_args'); + const useCookies = config_api.getConfigItem('ytdl_use_cookies'); + const is_audio = type === 'audio'; + + const fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; + + if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath; + + const customArgs = options.customArgs; + const customOutput = options.customOutput; + const customQualityConfiguration = options.customQualityConfiguration; + + // video-specific args + const selectedHeight = options.selectedHeight; + + // audio-specific args + const maxBitrate = options.maxBitrate; + + const youtubeUsername = options.youtubeUsername; + const youtubePassword = options.youtubePassword; + + let downloadConfig = null; + let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4']; + const is_youtube = url.includes('youtu'); + if (!is_audio && !is_youtube) { + // tiktok videos fail when using the default format + qualityPath = null; + } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) { + qualityPath = ['-f', 'bestvideo+bestaudio'] + } + + if (customArgs) { + downloadConfig = customArgs.split(',,'); + } else { + if (customQualityConfiguration) { + qualityPath = ['-f', customQualityConfiguration]; + } else if (selectedHeight && selectedHeight !== '' && !is_audio) { + qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; + } else if (is_audio) { + qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0'] + } + + if (customOutput) { + customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); + downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; + } else { + downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; + } + + if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath); + + if (is_audio && !options.skip_audio_args) { + downloadConfig.push('-x'); + downloadConfig.push('--audio-format', 'mp3'); + } + + if (youtubeUsername && youtubePassword) { + downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); + } + + if (useCookies) { + if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) { + downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); + } else { + logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); + } + } + + if (!useDefaultDownloadingAgent && customDownloadingAgent) { + downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); + } + + let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive) { + const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath; + const archive_path = path.join(archive_folder, `archive_${type}.txt`); + + await fs.ensureDir(archive_folder); + await fs.ensureFile(archive_path); + + let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); + await fs.ensureFile(blacklist_path); + + let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); + await fs.ensureFile(merged_path); + // merges blacklist and regular archive + let inputPathList = [archive_path, blacklist_path]; + let status = await mergeFiles(inputPathList, merged_path); + + options.merged_string = await fs.readFile(merged_path, "utf8"); + + downloadConfig.push('--download-archive', merged_path); + } + + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + downloadConfig.push('--write-thumbnail'); + } + + if (globalArgs && globalArgs !== '') { + // adds global args + if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) { + // if global args has an output, replce the original output with that of global args + const original_output_index = downloadConfig.indexOf('-o'); + downloadConfig.splice(original_output_index, 2); + } + downloadConfig = downloadConfig.concat(globalArgs.split(',,')); + } + + const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit'); + if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) { + downloadConfig.push('-r', rate_limit); + } + + const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + if (default_downloader === 'yt-dlp') { + downloadConfig.push('--no-clean-infojson'); + } + + } + + // filter out incompatible args + downloadConfig = filterArgs(downloadConfig, is_audio); + + logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); + return downloadConfig; +} + async function getVideoInfoByURL(url, args = [], download = null) { return new Promise(resolve => { // remove bad args @@ -281,4 +351,4 @@ async function getVideoInfoByURL(url, args = [], download = null) { } }); }); -} \ No newline at end of file +} diff --git a/backend/package-lock.json b/backend/package-lock.json index 57ee755..7949f7d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -2719,6 +2719,14 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "queue": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz", + "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==", + "requires": { + "inherits": "~2.0.3" + } + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", diff --git a/backend/package.json b/backend/package.json index b2cd7c1..b428882 100644 --- a/backend/package.json +++ b/backend/package.json @@ -59,6 +59,7 @@ "passport-local": "^1.0.0", "progress": "^2.0.3", "ps-node": "^0.1.6", + "queue": "^6.0.2", "read-last-lines": "^1.7.2", "shortid": "^2.2.15", "unzipper": "^0.10.10",