TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-16 18:01:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactored youtube-dl updating, added tests

Browse Source
This commit is contained in:
Isaac Abadi
2023-09-02 20:09:40 -04:00
parent 3615b810e4
commit 3fc1110fc1
4 changed files with 101 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/app.js
View File

@@ -670,7 +670,7 @@ async function getUrlInfos(url) {
async function startYoutubeDL() { async function startYoutubeDL() {
// auto update youtube-dl // auto update youtube-dl
youtubedl_api.verifyBinaryExistsLinux(); youtubedl_api.verifyBinaryExists();
const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
if (update_available) await youtubedl_api.updateYoutubeDL(update_available); if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
} }

3
backend/consts.js
View File

@@ -269,7 +269,8 @@ exports.AVAILABLE_PERMISSIONS = [
'tasks_manager' 'tasks_manager'
]; ];
exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details' exports.DETAILS_BIN_PATH = 'appdata/youtube-dl.json'
exports.OUTDATED_YOUTUBEDL_VERSION = "2020.00.00";
// args that have a value after it (e.g. -o <output> or -f <format>) // args that have a value after it (e.g. -o <output> or -f <format>)
const YTDL_ARGS_WITH_VALUES = [ const YTDL_ARGS_WITH_VALUES = [

34
backend/test/tests.js
View File

@@ -4,6 +4,9 @@ const low = require('lowdb')
const winston = require('winston'); const winston = require('winston');
const path = require('path'); const path = require('path');
const util = require('util'); const util = require('util');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
const exec = util.promisify(require('child_process').exec); const exec = util.promisify(require('child_process').exec);
const FileSync = require('lowdb/adapters/FileSync'); const FileSync = require('lowdb/adapters/FileSync');
@@ -44,9 +47,7 @@ const categories_api = require('../categories');
const files_api = require('../files'); const files_api = require('../files');
const youtubedl_api = require('../youtube-dl'); const youtubedl_api = require('../youtube-dl');
const config_api = require('../config'); const config_api = require('../config');
const fs = require('fs-extra'); const CONSTS = require('../consts');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
db_api.initialize(db, users_db, 'local_db_test.json'); db_api.initialize(db, users_db, 'local_db_test.json');
@@ -624,6 +625,33 @@ describe('Downloader', function() {
}); });
}); });
describe('youtube-dl', async function() {
beforeEach(async function () {
if (fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.unlinkSync(CONSTS.DETAILS_BIN_PATH);
await youtubedl_api.checkForYoutubeDLUpdate();
});
it('Check latest version', async function() {
const latest_version = await youtubedl_api.checkForYoutubeDLUpdate();
const default_details_bin = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
assert(default_details_bin['version'] === CONSTS.OUTDATED_YOUTUBEDL_VERSION);
assert(latest_version > default_details_bin['version']);
});
it('Update youtube-dl', async function() {
this.timeout(300000);
const original_fork = config_api.getConfigItem('ytdl_default_downloader');
const binary_path = path.join('test', 'test_binary');
for (const youtubedl_fork in youtubedl_api.youtubedl_forks) {
config_api.setConfigItem('ytdl_default_downloader', youtubedl_fork);
const latest_version = await youtubedl_api.checkForYoutubeDLUpdate();
await youtubedl_api.updateYoutubeDL(latest_version, binary_path);
assert(fs.existsSync(binary_path));
if (fs.existsSync(binary_path)) fs.unlinkSync(binary_path);
}
config_api.setConfigItem('ytdl_default_downloader', original_fork);
});
});
describe('Tasks', function() { describe('Tasks', function() {
const tasks_api = require('../tasks'); const tasks_api = require('../tasks');
beforeEach(async function() { beforeEach(async function() {

155
backend/youtube-dl.js
View File

@@ -7,22 +7,20 @@ const CONSTS = require('./consts');
const config_api = require('./config.js'); const config_api = require('./config.js');
const youtubedl = require('youtube-dl'); const youtubedl = require('youtube-dl');
const OUTDATED_VERSION = "2020.00.00";
const is_windows = process.platform === 'win32'; const is_windows = process.platform === 'win32';
const download_sources = { exports.youtubedl_forks = {
'youtube-dl': { 'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', 'download_url': 'https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl',
'func': downloadLatestYoutubeDLBinary 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'
}, },
'youtube-dlc': { 'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', 'download_url': 'https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc',
'func': downloadLatestYoutubeDLCBinary 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags'
}, },
'yt-dlp': { 'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', 'download_url': 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp',
'func': downloadLatestYoutubeDLPBinary 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags'
} }
} }
@@ -36,47 +34,78 @@ exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
} }
exports.checkForYoutubeDLUpdate = async () => { exports.checkForYoutubeDLUpdate = async () => {
return new Promise(async resolve => { const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); // get current version
const tags_url = download_sources[default_downloader]['tags_url']; let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
// get current version if (!current_app_details_exists) {
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
if (!current_app_details_exists) { fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": CONSTS.OUTDATED_YOUTUBEDL_VERSION, "downloader": default_downloader});
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); }
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader}); let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
} let current_version = current_app_details['version'];
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); let current_downloader = current_app_details['downloader'];
let current_version = current_app_details['version']; let stored_binary_path = current_app_details['path'];
let current_downloader = current_app_details['downloader']; if (!stored_binary_path || typeof stored_binary_path !== 'string') {
let stored_binary_path = current_app_details['path']; // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
if (!stored_binary_path || typeof stored_binary_path !== 'string') { const guessed_base_path = 'node_modules/youtube-dl/bin/';
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
const guessed_base_path = 'node_modules/youtube-dl/bin/'; if (fs.existsSync(guessed_file_path)) {
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); stored_binary_path = guessed_file_path;
if (fs.existsSync(guessed_file_path)) { // logger.info('INFO: Guess successful! Update process continuing...')
stored_binary_path = guessed_file_path; } else {
// logger.info('INFO: Guess successful! Update process continuing...') logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
} else { return null;
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(null);
return;
}
} }
}
// got version, now let's check the latest version from the youtube-dl API // got version, now let's check the latest version from the youtube-dl API
return await getLatestUpdateVersion(default_downloader, current_downloader, current_version)
}
exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path);
}
exports.verifyBinaryExists = () => {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
details_json['exec'] = 'youtube-dl';
details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
utils.restartServer();
}
}
async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) {
const file_ext = is_windows ? '.exe' : '';
// build the URL
const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
const output_path = custom_output_path || `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, youtubedl_fork);
}
const getLatestUpdateVersion = async (youtubedl_fork, current_downloader, current_version) => {
const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url'];
return new Promise(resolve => {
fetch(tags_url, {method: 'Get'}) fetch(tags_url, {method: 'Get'})
.then(async res => res.json()) .then(async res => res.json())
.then(async (json) => { .then(async (json) => {
// check if the versions are different // check if the versions are different
if (!json || !json[0]) { if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`) logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
resolve(null); resolve(null);
return; return;
} }
const latest_update_version = json[0]['name']; const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) { if (current_version !== latest_update_version ||
youtubedl_fork !== current_downloader) {
// versions different or different downloader is being used, download new update // versions different or different downloader is being used, download new update
resolve(latest_update_version); resolve(latest_update_version);
} else { } else {
@@ -85,7 +114,7 @@ exports.checkForYoutubeDLUpdate = async () => {
return; return;
}) })
.catch(err => { .catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`) logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
logger.error(err); logger.error(err);
resolve(null); resolve(null);
return; return;
@@ -93,56 +122,6 @@ exports.checkForYoutubeDLUpdate = async () => {
}); });
} }
exports.updateYoutubeDL = async (latest_update_version) => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await download_sources[default_downloader]['func'](latest_update_version);
}
exports.verifyBinaryExistsLinux = () => {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
details_json['exec'] = 'youtube-dl';
details_json['version'] = OUTDATED_VERSION;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
utils.restartServer();
}
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) { function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version; if (new_version) details_json['version'] = new_version;