TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-03 16:31:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Simplified youtube-dl run functions

Browse Source
This commit is contained in:
Isaac Abadi
2023-11-25 00:40:44 -05:00
parent 3a86d5c636
commit 9206d4ba28
4 changed files with 102 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/downloader.js
View File

@@ -293,7 +293,7 @@ exports.collectInfo = async (download_uid) => {
});
}
exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
@@ -323,7 +323,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = null) => {
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
const file_objs = [];
// download file
let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, customDownloadHandler);
if (child_process) download_to_child_process[download['uid']] = child_process;
const {parsed_output, err} = await callback;
clearInterval(download_checker);
@@ -568,7 +568,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
new_args.push('--dump-json');
const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(url, new_args);
let {callback} = await youtubedl_api.runYoutubeDL(url, new_args);
const {parsed_output, err} = await callback;
if (!parsed_output || parsed_output.length === 0) {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;

11
backend/subscriptions.js
View File

@@ -63,7 +63,8 @@ async function getSubscriptionInfo(sub) {
}
}
const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
const {parsed_output, err} = await callback;
if (err) {
logger.error(err.stderr);
return false;
@@ -225,8 +226,9 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
// get videos
logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
const {parsed_output, err} = await callback;
updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid);
if (!parsed_output) {
logger.error('Subscription check failed!');
if (err) logger.error(err);
@@ -480,7 +482,8 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
if (info[metric_to_compare] > file_obj[metric_to_compare]) {
// download new video as the simulated one is better
const {parsed_output, err} = await youtubedl_api.runYoutubeDLMain(sub.url, downloadConfig);
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
const {parsed_output, err} = await callback;
if (err) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (parsed_output) {

63
backend/test/tests.js
View File

@@ -662,6 +662,69 @@ describe('youtube-dl', async function() {
});
});
describe('Subscriptions', function() {
const new_sub = {
name: 'test_sub',
url: 'https://www.youtube.com/channel/UCzofo-P8yMMCOv8rsPfIR-g',
maxQuality: null,
id: uuid(),
user_uid: null,
type: 'video',
paused: true
};
beforeEach(async function() {
await db_api.removeAllRecords('subscriptions');
});
it('Subscribe', async function () {
const success = await subscriptions_api.subscribe(new_sub, null, true);
assert(success);
const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
assert(sub_exists);
});
it('Unsubscribe', async function () {
await subscriptions_api.subscribe(new_sub, null, true);
await subscriptions_api.unsubscribe(new_sub);
const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
assert(!sub_exists);
});
it('Delete subscription file', async function () {
});
it('Get subscription by name', async function () {
await subscriptions_api.subscribe(new_sub, null, true);
const sub_by_name = await subscriptions_api.getSubscriptionByName('test_sub');
assert(sub_by_name);
});
it('Get subscriptions', async function() {
await subscriptions_api.subscribe(new_sub, null, true);
const subs = await subscriptions_api.getSubscriptions(null);
assert(subs && subs.length === 1);
});
it('Update subscription', async function () {
await subscriptions_api.subscribe(new_sub, null, true);
const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
await subscriptions_api.updateSubscription(sub_update);
const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
assert(updated_sub['name'] === 'updated_name');
});
it('Update subscription property', async function () {
await subscriptions_api.subscribe(new_sub, null, true);
const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
await subscriptions_api.updateSubscriptionPropertyMultiple([sub_update], {name: 'updated_name'});
const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
assert(updated_sub['name'] === 'updated_name');
});
it('Write subscription metadata', async function() {
const metadata_path = path.join('subscriptions', 'channels', 'test_sub', 'subscription_backup.json');
if (fs.existsSync(metadata_path)) fs.unlinkSync(metadata_path);
await subscriptions_api.subscribe(new_sub, null, true);
assert(fs.existsSync(metadata_path));
});
it('Fresh uploads', async function() {
});
});
describe('Tasks', function() {
const tasks_api = require('../tasks');
beforeEach(async function() {

52
backend/youtube-dl.js
View File

@@ -1,5 +1,6 @@
const fs = require('fs-extra');
const fetch = require('node-fetch');
const path = require('path');
const execa = require('execa');
const kill = require('tree-kill');
@@ -25,11 +26,11 @@ exports.youtubedl_forks = {
}
}
exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => {
let callback = null;
let child_process = null;
if (downloadMethod) {
callback = exports.runYoutubeDLMain(url, args, downloadMethod);
if (customDownloadHandler) {
callback = runYoutubeDLCustom(url, args, customDownloadHandler);
} else {
({callback, child_process} = await runYoutubeDLProcess(url, args));
}
@@ -37,19 +38,27 @@ exports.runYoutubeDL = async (url, args, downloadMethod = null) => {
return {child_process, callback};
}
// Run youtube-dl in a main thread (with possible downloadMethod)
exports.runYoutubeDLMain = async (url, args, downloadMethod = defaultDownloadMethod) => {
// Run youtube-dl directly (not cancellable)
const runYoutubeDLCustom = async (url, args, customDownloadHandler) => {
const downloadHandler = customDownloadHandler;
return new Promise(resolve => {
downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) {
const parsed_output = utils.parseOutputJSON(output, err);
resolve({parsed_output, err});
});
});
}
// Run youtube-dl in a subprocess
const runYoutubeDLProcess = async (url, args) => {
const child_process = execa(await getYoutubeDLPath(), [url, ...args], {maxBuffer: Infinity});
// Run youtube-dl in a subprocess (cancellable)
const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => {
const youtubedl_path = getYoutubeDLPath(youtubedl_fork);
const binary_exists = fs.existsSync(youtubedl_path);
if (!binary_exists) {
const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`;
logger.error(err);
return;
}
const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity});
const callback = new Promise(async resolve => {
try {
const {stdout, stderr} = await child_process;
@@ -62,15 +71,10 @@ const runYoutubeDLProcess = async (url, args) => {
return {child_process, callback}
}
async function getYoutubeDLPath() {
const guessed_base_path = 'node_modules/youtube-dl/bin/';
return guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
}
async function defaultDownloadMethod(url, args, options, callback) {
const {child_process, _} = await runYoutubeDLProcess(url, args);
const {stdout, stderr} = await child_process;
return await callback(stderr, stdout.trim().split(/\r?\n/));
function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) {
const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : '');
const binary_path = path.join('appdata', 'bin', binary_file_name);
return binary_path;
}
exports.killYoutubeDLProcess = async (child_process) => {
@@ -91,14 +95,13 @@ exports.checkForYoutubeDLUpdate = async () => {
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
const guessed_file_path = getYoutubeDLPath();
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
return null;
logger.warn(`youtuble-dl binary not found at '${guessed_file_path}', downloading...`);
return true;
}
}
@@ -107,6 +110,7 @@ exports.checkForYoutubeDLUpdate = async () => {
}
exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => {
await fs.ensureDir(path.join('appdata', 'bin'));
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path);
}
@@ -114,7 +118,7 @@ exports.updateYoutubeDL = async (latest_update_version, custom_output_path = nul
exports.verifyBinaryExists = () => {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
details_json['path'] = getYoutubeDLPath();
details_json['exec'] = 'youtube-dl';
details_json['version'] = CONSTS.OUTDATED_YOUTUBEDL_VERSION;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
@@ -128,7 +132,7 @@ async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version,
// build the URL
const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
const output_path = custom_output_path || `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork);
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);