From 9ca6563b360485749c61ad76a37b6f751f9ed9a1 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Sun, 26 Nov 2023 19:10:22 -0500 Subject: [PATCH] Multiple binaries can now exist, and if the needed one is missing it will be downloaded at runtime --- backend/app.js | 7 +--- backend/downloader.js | 2 +- backend/subscriptions.js | 2 +- backend/utils.js | 5 --- backend/youtube-dl.js | 76 ++++++++++++++-------------------------- 5 files changed, 30 insertions(+), 62 deletions(-) diff --git a/backend/app.js b/backend/app.js index 967688f..f1cb8e6 100644 --- a/backend/app.js +++ b/backend/app.js @@ -20,9 +20,6 @@ const ps = require('ps-node'); const Feed = require('feed').Feed; const session = require('express-session'); -// needed if bin/details somehow gets deleted -if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) - const logger = require('./logger'); const config_api = require('./config.js'); const downloader_api = require('./downloader'); @@ -668,9 +665,7 @@ async function getUrlInfos(url) { async function startYoutubeDL() { // auto update youtube-dl - youtubedl_api.verifyBinaryExists(); - const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); - if (update_available) await youtubedl_api.updateYoutubeDL(update_available); + await youtubedl_api.checkForYoutubeDLUpdate(); } app.use(function(req, res, next) { diff --git a/backend/downloader.js b/backend/downloader.js index cebc87d..c23b23e 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -426,7 +426,7 @@ exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) = // helper functions exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => { - const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) { logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.') diff --git a/backend/subscriptions.js b/backend/subscriptions.js index cca0c6c..034d652 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -374,7 +374,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push('-r', rate_limit); } - const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); if (default_downloader === 'yt-dlp') { downloadConfig.push('--no-clean-info-json'); } diff --git a/backend/utils.js b/backend/utils.js index 3c7b794..27d0f66 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -241,11 +241,6 @@ exports.addUIDsToCategory = (category, files) => { return files_that_match; } -exports.getCurrentDownloader = () => { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - return details_json['downloader']; -} - exports.recFindByExt = async (base, ext, files, result, recursive = true) => { files = files || (await fs.readdir(base)) result = result || [] diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js index e75692c..a999628 100644 --- a/backend/youtube-dl.js +++ b/backend/youtube-dl.js @@ -27,6 +27,8 @@ exports.youtubedl_forks = { } exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => { + const output_file_path = getYoutubeDLPath(); + if (!fs.existsSync(output_file_path)) await exports.checkForYoutubeDLUpdate(); let callback = null; let child_process = null; if (customDownloadHandler) { @@ -82,31 +84,25 @@ exports.killYoutubeDLProcess = async (child_process) => { } exports.checkForYoutubeDLUpdate = async () => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + const selected_fork = config_api.getConfigItem('ytdl_default_downloader'); + const output_file_path = getYoutubeDLPath(); // get current version let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); if (!current_app_details_exists) { logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": CONSTS.OUTDATED_YOUTUBEDL_VERSION, "downloader": default_downloader}); - } - let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); - let current_version = current_app_details['version']; - let current_downloader = current_app_details['downloader']; - let stored_binary_path = current_app_details['path']; - if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_file_path = getYoutubeDLPath(); - if (fs.existsSync(guessed_file_path)) { - stored_binary_path = guessed_file_path; - // logger.info('INFO: Guess successful! Update process continuing...') - } else { - logger.warn(`youtuble-dl binary not found at '${guessed_file_path}', downloading...`); - return true; - } + updateDetailsJSON(CONSTS.OUTDATED_YOUTUBEDL_VERSION, selected_fork, output_file_path); } + const current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); + const current_version = current_app_details['version']; + const current_fork = current_app_details['downloader']; - // got version, now let's check the latest version from the youtube-dl API - return await getLatestUpdateVersion(default_downloader, current_downloader, current_version) + const latest_version = await getLatestUpdateVersion(selected_fork); + // if the binary does not exist, or default_downloader doesn't match existing fork, or if the fork has been updated, redownload + // TODO: don't redownload if fork already exists + if (!fs.existsSync(output_file_path) || current_fork !== selected_fork || !current_version || current_version !== latest_version) { + logger.warn(`Updating ${selected_fork} binary to '${output_file_path}', downloading...`); + exports.updateYoutubeDL(latest_version); + } } exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => { @@ -115,18 +111,6 @@ exports.updateYoutubeDL = async (latest_update_version, custom_output_path = nul await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path); } -exports.verifyBinaryExists = () => { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) { - details_json['path'] = getYoutubeDLPath(); - details_json['exec'] = 'youtube-dl'; - details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION; - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); - - utils.restartServer(); - } -} - async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) { const file_ext = is_windows ? '.exe' : ''; @@ -134,48 +118,42 @@ async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`; const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork); - await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); + await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`); + fs.chmod(output_path, 0o777); updateDetailsJSON(new_version, youtubedl_fork, output_path); -} +} -const getLatestUpdateVersion = async (youtubedl_fork, current_downloader, current_version) => { +const getLatestUpdateVersion = async (youtubedl_fork) => { const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url']; return new Promise(resolve => { fetch(tags_url, {method: 'Get'}) .then(async res => res.json()) .then(async (json) => { - // check if the versions are different if (!json || !json[0]) { logger.error(`Failed to check ${youtubedl_fork} version for an update.`) resolve(null); return; } const latest_update_version = json[0]['name']; - if (current_version !== latest_update_version || - youtubedl_fork !== current_downloader) { - // versions different or different downloader is being used, download new update - resolve(latest_update_version); - } else { - resolve(null); - } - return; + resolve(latest_update_version); }) .catch(err => { logger.error(`Failed to check ${youtubedl_fork} version for an update.`) logger.error(err); resolve(null); - return; }); }); } function updateDetailsJSON(new_version, fork, output_path) { const file_ext = is_windows ? '.exe' : ''; - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (new_version) details_json['version'] = new_version; - details_json['downloader'] = fork; - details_json['path'] = output_path; // unused - details_json['exec'] = fork + file_ext; // unused + const details_json = fs.existsSync(CONSTS.DETAILS_BIN_PATH) ? fs.readJSONSync(CONSTS.DETAILS_BIN_PATH) : {}; + if (!details_json[fork]) details_json[fork] = {}; + const fork_json = details_json[fork]; + fork_json['version'] = new_version; + fork_json['downloader'] = fork; + fork_json['path'] = output_path; // unused + fork_json['exec'] = fork + file_ext; // unused fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); }