diff --git a/backend/app.js b/backend/app.js index f3c08d5..a8c115c 100644 --- a/backend/app.js +++ b/backend/app.js @@ -670,7 +670,7 @@ async function getUrlInfos(url) { async function startYoutubeDL() { // auto update youtube-dl - youtubedl_api.verifyBinaryExistsLinux(); + youtubedl_api.verifyBinaryExists(); const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); if (update_available) await youtubedl_api.updateYoutubeDL(update_available); } diff --git a/backend/consts.js b/backend/consts.js index e121dad..b092ca5 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -269,7 +269,8 @@ exports.AVAILABLE_PERMISSIONS = [ 'tasks_manager' ]; -exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details' +exports.DETAILS_BIN_PATH = 'appdata/youtube-dl.json' +exports.OUTDATED_YOUTUBEDL_VERSION = "2020.00.00"; // args that have a value after it (e.g. -o or -f ) const YTDL_ARGS_WITH_VALUES = [ diff --git a/backend/test/tests.js b/backend/test/tests.js index 560e925..e1bd3b2 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -4,6 +4,9 @@ const low = require('lowdb') const winston = require('winston'); const path = require('path'); const util = require('util'); +const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); +const NodeID3 = require('node-id3'); const exec = util.promisify(require('child_process').exec); const FileSync = require('lowdb/adapters/FileSync'); @@ -44,9 +47,7 @@ const categories_api = require('../categories'); const files_api = require('../files'); const youtubedl_api = require('../youtube-dl'); const config_api = require('../config'); -const fs = require('fs-extra'); -const { uuid } = require('uuidv4'); -const NodeID3 = require('node-id3'); +const CONSTS = require('../consts'); db_api.initialize(db, users_db, 'local_db_test.json'); @@ -624,6 +625,33 @@ describe('Downloader', function() { }); }); +describe('youtube-dl', async function() { + beforeEach(async function () { + if (fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.unlinkSync(CONSTS.DETAILS_BIN_PATH); + await youtubedl_api.checkForYoutubeDLUpdate(); + }); + it('Check latest version', async function() { + const latest_version = await youtubedl_api.checkForYoutubeDLUpdate(); + const default_details_bin = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); + assert(default_details_bin['version'] === CONSTS.OUTDATED_YOUTUBEDL_VERSION); + assert(latest_version > default_details_bin['version']); + }); + + it('Update youtube-dl', async function() { + this.timeout(300000); + const original_fork = config_api.getConfigItem('ytdl_default_downloader'); + const binary_path = path.join('test', 'test_binary'); + for (const youtubedl_fork in youtubedl_api.youtubedl_forks) { + config_api.setConfigItem('ytdl_default_downloader', youtubedl_fork); + const latest_version = await youtubedl_api.checkForYoutubeDLUpdate(); + await youtubedl_api.updateYoutubeDL(latest_version, binary_path); + assert(fs.existsSync(binary_path)); + if (fs.existsSync(binary_path)) fs.unlinkSync(binary_path); + } + config_api.setConfigItem('ytdl_default_downloader', original_fork); + }); +}); + describe('Tasks', function() { const tasks_api = require('../tasks'); beforeEach(async function() { diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js index c951589..6bea878 100644 --- a/backend/youtube-dl.js +++ b/backend/youtube-dl.js @@ -7,22 +7,20 @@ const CONSTS = require('./consts'); const config_api = require('./config.js'); const youtubedl = require('youtube-dl'); -const OUTDATED_VERSION = "2020.00.00"; - const is_windows = process.platform === 'win32'; -const download_sources = { +exports.youtubedl_forks = { 'youtube-dl': { - 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', - 'func': downloadLatestYoutubeDLBinary + 'download_url': 'https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl', + 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags' }, 'youtube-dlc': { - 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', - 'func': downloadLatestYoutubeDLCBinary + 'download_url': 'https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc', + 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags' }, 'yt-dlp': { - 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', - 'func': downloadLatestYoutubeDLPBinary + 'download_url': 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp', + 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags' } } @@ -36,47 +34,78 @@ exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => { } exports.checkForYoutubeDLUpdate = async () => { - return new Promise(async resolve => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - const tags_url = download_sources[default_downloader]['tags_url']; - // get current version - let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); - if (!current_app_details_exists) { - logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader}); - } - let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); - let current_version = current_app_details['version']; - let current_downloader = current_app_details['downloader']; - let stored_binary_path = current_app_details['path']; - if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); - if (fs.existsSync(guessed_file_path)) { - stored_binary_path = guessed_file_path; - // logger.info('INFO: Guess successful! Update process continuing...') - } else { - logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); - resolve(null); - return; - } + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + // get current version + let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); + if (!current_app_details_exists) { + logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": CONSTS.OUTDATED_YOUTUBEDL_VERSION, "downloader": default_downloader}); + } + let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); + let current_version = current_app_details['version']; + let current_downloader = current_app_details['downloader']; + let stored_binary_path = current_app_details['path']; + if (!stored_binary_path || typeof stored_binary_path !== 'string') { + // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); + const guessed_base_path = 'node_modules/youtube-dl/bin/'; + const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); + if (fs.existsSync(guessed_file_path)) { + stored_binary_path = guessed_file_path; + // logger.info('INFO: Guess successful! Update process continuing...') + } else { + logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); + return null; } + } - // got version, now let's check the latest version from the youtube-dl API + // got version, now let's check the latest version from the youtube-dl API + return await getLatestUpdateVersion(default_downloader, current_downloader, current_version) +} +exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => { + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path); +} +exports.verifyBinaryExists = () => { + const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); + if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) { + details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl'; + details_json['exec'] = 'youtube-dl'; + details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION; + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); + + utils.restartServer(); + } +} + +async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) { + const file_ext = is_windows ? '.exe' : ''; + + // build the URL + const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`; + const output_path = custom_output_path || `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); + + updateDetailsJSON(new_version, youtubedl_fork); +} + +const getLatestUpdateVersion = async (youtubedl_fork, current_downloader, current_version) => { + const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url']; + return new Promise(resolve => { fetch(tags_url, {method: 'Get'}) .then(async res => res.json()) .then(async (json) => { // check if the versions are different if (!json || !json[0]) { - logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(`Failed to check ${youtubedl_fork} version for an update.`) resolve(null); return; } const latest_update_version = json[0]['name']; - if (current_version !== latest_update_version || default_downloader !== current_downloader) { + if (current_version !== latest_update_version || + youtubedl_fork !== current_downloader) { // versions different or different downloader is being used, download new update resolve(latest_update_version); } else { @@ -85,7 +114,7 @@ exports.checkForYoutubeDLUpdate = async () => { return; }) .catch(err => { - logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(`Failed to check ${youtubedl_fork} version for an update.`) logger.error(err); resolve(null); return; @@ -93,56 +122,6 @@ exports.checkForYoutubeDLUpdate = async () => { }); } -exports.updateYoutubeDL = async (latest_update_version) => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - await download_sources[default_downloader]['func'](latest_update_version); -} - -exports.verifyBinaryExistsLinux = () => { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) { - details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl'; - details_json['exec'] = 'youtube-dl'; - details_json['version'] = OUTDATED_VERSION; - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); - - utils.restartServer(); - } -} - -async function downloadLatestYoutubeDLBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dl'); -} - -async function downloadLatestYoutubeDLCBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dlc'); -} - -async function downloadLatestYoutubeDLPBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`); - - updateDetailsJSON(new_version, 'yt-dlp'); -} - function updateDetailsJSON(new_version, downloader) { const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); if (new_version) details_json['version'] = new_version;