TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-13 10:41:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Consolidated all youtube-dl calls into one function

Browse Source
This commit is contained in:
Tzahi12345
2023-06-06 01:20:36 -04:00
parent d3cb957991
commit cd0aaf6a61
3 changed files with 91 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
backend/app.js
View File

@@ -657,27 +657,13 @@ function generateEnvVarConfigItem(key) {
// currently only works for single urls // currently only works for single urls
async function getUrlInfos(url) { async function getUrlInfos(url) {
let startDate = Date.now(); const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, ['--dump-json']);
let result = []; if (!parsed_output || parsed_output.length !== 1) {
return new Promise(resolve => { logger.error(`Failed to retrieve available formats for url: ${url}`);
youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { if (err) logger.error(err);
let new_date = Date.now(); return null;
let difference = (new_date - startDate)/1000; }
logger.debug(`URL info retrieval delay: ${difference} seconds.`); return parsed_output[0];
if (err) {
logger.error(`Error during retrieving formats for ${url}: ${err}`);
resolve(null);
}
let try_putput = null;
try {
try_putput = JSON.parse(output);
result = try_putput;
} catch(e) {
logger.error(`Failed to retrieve available formats for url: ${url}`);
}
resolve(result);
});
});
} }
// youtube-dl functions // youtube-dl functions

68
backend/downloader.js
View File

@@ -187,7 +187,7 @@ async function checkDownloads() {
// move to next step // move to next step
running_downloads_count++; running_downloads_count++;
if (waiting_download['step_index'] === 0) { if (waiting_download['step_index'] === 0) {
collectInfo(waiting_download['uid']); exports.collectInfo(waiting_download['uid']);
} else if (waiting_download['step_index'] === 1) { } else if (waiting_download['step_index'] === 1) {
exports.downloadQueuedFile(waiting_download['uid']); exports.downloadQueuedFile(waiting_download['uid']);
} }
@@ -195,7 +195,7 @@ async function checkDownloads() {
} }
} }
async function collectInfo(download_uid) { exports.collectInfo = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid}); const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) { if (download['paused']) {
return; return;
@@ -218,17 +218,18 @@ async function collectInfo(download_uid) {
// get video info prior to download // get video info prior to download
let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid); let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
if (!info) { if (!info || info.length === 0) {
// info failed, error presumably already recorded // info failed, error presumably already recorded
return; return;
} }
// in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) { if (useYoutubeDLArchive && !options.ignoreArchive && info.length === 1) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']); const info_obj = info[0];
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info_obj['id'], type, download['user_uid'], download['sub_id']);
if (exists_in_archive) { if (exists_in_archive) {
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`; const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
logger.warn(error); logger.warn(error);
if (download_uid) { if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid}); const download = await db_api.getRecord('download_queue', {uid: download_uid});
@@ -241,7 +242,7 @@ async function collectInfo(download_uid) {
let category = null; let category = null;
// check if it fits into a category. If so, then get info again using new args // check if it fits into a category. If so, then get info again using new args
if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); if (info.length === 1 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name // set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) { if (category && category['custom_output']) {
@@ -262,14 +263,14 @@ async function collectInfo(download_uid) {
// store info in download for future use // store info in download for future use
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename'])); for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null; const title = info.length > 1 ? info[0]['playlist_title'] || info[0]['playlist'] : info[0]['title'];
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args, await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
finished_step: true, finished_step: true,
running: false, running: false,
options: options, options: options,
files_to_check_for_progress: files_to_check_for_progress, files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size, expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title'], title: title,
category: stripped_category, category: stripped_category,
prefetched_info: null prefetched_info: null
}); });
@@ -385,8 +386,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
if (file_objs.length > 1) { if (file_objs.length > 1) {
// create playlist // create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); container = await files_api.createPlaylist(download['title'], file_objs.map(file_obj => file_obj.uid), download['user_uid']);
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
} else if (file_objs.length === 1) { } else if (file_objs.length === 1) {
container = file_objs[0]; container = file_objs[0];
} else { } else {
@@ -536,34 +536,30 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
} }
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
return new Promise(resolve => { // remove bad args
// remove bad args const temp_args = utils.filterArgs(args, ['--no-simulate']);
const temp_args = utils.filterArgs(args, ['--no-simulate']); const new_args = [...temp_args];
const new_args = [...temp_args];
const archiveArgIndex = new_args.indexOf('--download-archive'); const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) { if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2); new_args.splice(archiveArgIndex, 2);
}
new_args.push('--dump-json');
const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, new_args);
if (!parsed_output || parsed_output.length === 0) {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error_message, 'info_retrieve_failed');
} }
return null;
}
new_args.push('--dump-json'); return parsed_output;
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
const parsed_output = utils.parseOutputJSON(output, err);
if (parsed_output) {
resolve(parsed_output);
} else {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error_message, 'info_retrieve_failed');
}
resolve(null);
}
});
});
} }
function filterArgs(args, isAudio) { function filterArgs(args, isAudio) {

129
backend/subscriptions.js
View File

@@ -2,6 +2,7 @@ const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const youtubedl = require('youtube-dl'); const youtubedl = require('youtube-dl');
const youtubedl_api = require('./youtube-dl');
const config_api = require('./config'); const config_api = require('./config');
const archive_api = require('./archive'); const archive_api = require('./archive');
const utils = require('./utils'); const utils = require('./utils');
@@ -63,52 +64,36 @@ async function getSubscriptionInfo(sub) {
} }
} }
return new Promise(async resolve => { const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => { if (err) {
if (debugMode) { logger.error(err.stderr);
logger.info('Subscribe: got info for subscription ' + sub.id); return false;
} }
if (err) { logger.verbose('Subscribe: got info for subscription ' + sub.id);
logger.error(err.stderr); for (const output_json of parsed_output) {
resolve(false); if (!output_json) {
} else if (output) { continue;
if (output.length === 0 || (output.length === 1 && output[0] === '')) { }
logger.verbose('Could not get info for ' + sub.id);
resolve(false);
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
if (!sub.name) {
if (sub.isPlaylist) {
sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
} else {
sub.name = output_json.uploader;
}
// if it's now valid, update
if (sub.name) {
let sub_name = sub.name;
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
if (sub_name_exists) sub_name += ` - ${sub.id}`;
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
}
}
// TODO: get even more info if (!sub.name) {
if (sub.isPlaylist) {
resolve(true); sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
} } else {
resolve(false); sub.name = output_json.uploader;
} }
}); // if it's now valid, update
}); if (sub.name) {
let sub_name = sub.name;
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
if (sub_name_exists) sub_name += ` - ${sub.id}`;
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
}
}
return true;
}
return false;
} }
exports.unsubscribe = async (sub, deleteMode, user_uid = null) => { exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
@@ -241,33 +226,24 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
// get videos // get videos
logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`); logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
return new Promise(async resolve => { const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { updateSubscriptionProperty(sub, {downloading: false}, user_uid);
// cleanup if (!parsed_output) {
updateSubscriptionProperty(sub, {downloading: false}, user_uid); logger.error('Subscription check failed!');
if (err) logger.error(err);
return null;
}
// remove temporary archive file if it exists // remove temporary archive file if it exists
const archive_path = path.join(appendedBasePath, 'archive.txt'); const archive_path = path.join(appendedBasePath, 'archive.txt');
const archive_exists = await fs.pathExists(archive_path); const archive_exists = await fs.pathExists(archive_path);
if (archive_exists) { if (archive_exists) {
await fs.unlink(archive_path); await fs.unlink(archive_path);
} }
logger.verbose('Subscription: finished check for ' + sub.name); logger.verbose('Subscription: finished check for ' + sub.name);
const parsed_output = utils.parseOutputJSON(output, err); const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
if (!parsed_output) { return files_to_download;
logger.error('Subscription check failed!');
resolve(null);
return;
}
const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
resolve(files_to_download);
return;
});
}, err => {
logger.error(err);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
});
} }
async function handleOutputJSON(output_jsons, sub, user_uid) { async function handleOutputJSON(output_jsons, sub, user_uid) {
@@ -506,14 +482,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
if (output[metric_to_compare] > file_obj[metric_to_compare]) { if (output[metric_to_compare] > file_obj[metric_to_compare]) {
// download new video as the simulated one is better // download new video as the simulated one is better
youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => { const {parsed_output, err} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
if (err) { if (err) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`); logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (output) { } else if (parsed_output) {
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}); await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
} }
});
} }
} }
}); });