mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-07 12:00:01 +03:00
youtube-dl refactor (#956)
* Consolidated all youtube-dl calls into one function * Downloads can now be cancelled and better "paused" * Removed node-youtube-dl dependency * Added ability to manually check a subscription, and to cancel a subscription check --------- Co-authored-by: Dedy Martadinata S <dedyms@proton.me>
This commit is contained in:
Tzahi12345
GitHub
@@ -1,7 +1,7 @@
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const youtubedl = require('youtube-dl');
|
||||
|
||||
const youtubedl_api = require('./youtube-dl');
|
||||
const config_api = require('./config');
|
||||
const archive_api = require('./archive');
|
||||
const utils = require('./utils');
|
||||
@@ -39,7 +39,7 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
|
||||
exports.writeSubscriptionMetadata(sub);
|
||||
|
||||
if (success) {
|
||||
if (!sub.paused) exports.getVideosForSub(sub, user_uid);
|
||||
if (!sub.paused) exports.getVideosForSub(sub.id);
|
||||
} else {
|
||||
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
|
||||
}
|
||||
@@ -63,55 +63,41 @@ async function getSubscriptionInfo(sub) {
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise(async resolve => {
|
||||
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
|
||||
if (debugMode) {
|
||||
logger.info('Subscribe: got info for subscription ' + sub.id);
|
||||
}
|
||||
if (err) {
|
||||
logger.error(err.stderr);
|
||||
resolve(false);
|
||||
} else if (output) {
|
||||
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
|
||||
logger.verbose('Could not get info for ' + sub.id);
|
||||
resolve(false);
|
||||
}
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
let output_json = null;
|
||||
try {
|
||||
output_json = JSON.parse(output[i]);
|
||||
} catch(e) {
|
||||
output_json = null;
|
||||
}
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
if (!sub.name) {
|
||||
if (sub.isPlaylist) {
|
||||
sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
|
||||
} else {
|
||||
sub.name = output_json.uploader;
|
||||
}
|
||||
// if it's now valid, update
|
||||
if (sub.name) {
|
||||
let sub_name = sub.name;
|
||||
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
|
||||
if (sub_name_exists) sub_name += ` - ${sub.id}`;
|
||||
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
|
||||
}
|
||||
}
|
||||
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
||||
const {parsed_output, err} = await callback;
|
||||
if (err) {
|
||||
logger.error(err.stderr);
|
||||
return false;
|
||||
}
|
||||
logger.verbose('Subscribe: got info for subscription ' + sub.id);
|
||||
for (const output_json of parsed_output) {
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: get even more info
|
||||
|
||||
resolve(true);
|
||||
}
|
||||
resolve(false);
|
||||
if (!sub.name) {
|
||||
if (sub.isPlaylist) {
|
||||
sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
|
||||
} else {
|
||||
sub.name = output_json.uploader;
|
||||
}
|
||||
});
|
||||
});
|
||||
// if it's now valid, update
|
||||
if (sub.name) {
|
||||
let sub_name = sub.name;
|
||||
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
|
||||
if (sub_name_exists) sub_name += ` - ${sub.id}`;
|
||||
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
|
||||
exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => {
|
||||
const sub = await exports.getSubscription(sub_id);
|
||||
let basePath = null;
|
||||
if (user_uid)
|
||||
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
|
||||
@@ -134,6 +120,7 @@ exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
|
||||
}
|
||||
}
|
||||
|
||||
await killSubDownloads(sub_id, true);
|
||||
await db_api.removeRecord('subscriptions', {id: id});
|
||||
await db_api.removeAllRecords('files', {sub_id: id});
|
||||
|
||||
@@ -218,12 +205,18 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
|
||||
}
|
||||
}
|
||||
|
||||
exports.getVideosForSub = async (sub, user_uid = null) => {
|
||||
const latest_sub_obj = await exports.getSubscription(sub.id);
|
||||
if (!latest_sub_obj || latest_sub_obj['downloading']) {
|
||||
exports.getVideosForSub = async (sub_id) => {
|
||||
const sub = await exports.getSubscription(sub_id);
|
||||
if (!sub || sub['downloading']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
_getVideosForSub(sub);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function _getVideosForSub(sub) {
|
||||
const user_uid = sub['user_uid'];
|
||||
updateSubscriptionProperty(sub, {downloading: true}, user_uid);
|
||||
|
||||
// get basePath
|
||||
@@ -241,33 +234,26 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
|
||||
// get videos
|
||||
logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
|
||||
|
||||
return new Promise(async resolve => {
|
||||
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
|
||||
// cleanup
|
||||
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
|
||||
let {child_process, callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
||||
updateSubscriptionProperty(sub, {child_process: child_process}, user_uid);
|
||||
const {parsed_output, err} = await callback;
|
||||
updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid);
|
||||
if (!parsed_output) {
|
||||
logger.error('Subscription check failed!');
|
||||
if (err) logger.error(err);
|
||||
return null;
|
||||
}
|
||||
|
||||
// remove temporary archive file if it exists
|
||||
const archive_path = path.join(appendedBasePath, 'archive.txt');
|
||||
const archive_exists = await fs.pathExists(archive_path);
|
||||
if (archive_exists) {
|
||||
await fs.unlink(archive_path);
|
||||
}
|
||||
// remove temporary archive file if it exists
|
||||
const archive_path = path.join(appendedBasePath, 'archive.txt');
|
||||
const archive_exists = await fs.pathExists(archive_path);
|
||||
if (archive_exists) {
|
||||
await fs.unlink(archive_path);
|
||||
}
|
||||
|
||||
logger.verbose('Subscription: finished check for ' + sub.name);
|
||||
const parsed_output = utils.parseOutputJSON(output, err);
|
||||
if (!parsed_output) {
|
||||
logger.error('Subscription check failed!');
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
|
||||
resolve(files_to_download);
|
||||
return;
|
||||
});
|
||||
}, err => {
|
||||
logger.error(err);
|
||||
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
|
||||
});
|
||||
logger.verbose('Subscription: finished check for ' + sub.name);
|
||||
const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
|
||||
return files_to_download;
|
||||
}
|
||||
|
||||
async function handleOutputJSON(output_jsons, sub, user_uid) {
|
||||
@@ -388,7 +374,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
|
||||
downloadConfig.push('-r', rate_limit);
|
||||
}
|
||||
|
||||
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
|
||||
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
||||
if (default_downloader === 'yt-dlp') {
|
||||
downloadConfig.push('--no-clean-info-json');
|
||||
}
|
||||
@@ -419,8 +405,37 @@ async function getFilesToDownload(sub, output_jsons) {
|
||||
return files_to_download;
|
||||
}
|
||||
|
||||
exports.cancelCheckSubscription = async (sub_id) => {
|
||||
const sub = await exports.getSubscription(sub_id);
|
||||
if (!sub['downloading'] && !sub['child_process']) {
|
||||
logger.error('Failed to cancel subscription check, verify that it is still running!');
|
||||
return false;
|
||||
}
|
||||
|
||||
// if check is ongoing
|
||||
if (sub['child_process']) {
|
||||
const child_process = sub['child_process'];
|
||||
youtubedl_api.killYoutubeDLProcess(child_process);
|
||||
}
|
||||
|
||||
// cancel activate video downloads
|
||||
await killSubDownloads(sub_id);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function killSubDownloads(sub_id, remove_downloads = false) {
|
||||
const sub_downloads = await db_api.getRecords('download_queue', {sub_id: sub_id});
|
||||
for (const sub_download of sub_downloads) {
|
||||
if (sub_download['running'])
|
||||
await downloader_api.cancelDownload(sub_download['uid']);
|
||||
if (remove_downloads)
|
||||
await db_api.removeRecord('download_queue', {uid: sub_download['uid']});
|
||||
}
|
||||
}
|
||||
|
||||
exports.getSubscriptions = async (user_uid = null) => {
|
||||
// TODO: fix issue where the downloading property may not match getSubscription()
|
||||
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
|
||||
}
|
||||
|
||||
@@ -499,24 +514,22 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
|
||||
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
|
||||
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
|
||||
// simulate a download to verify that a better version exists
|
||||
youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
|
||||
if (err) {
|
||||
// video is not available anymore for whatever reason
|
||||
} else if (output) {
|
||||
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
|
||||
if (output[metric_to_compare] > file_obj[metric_to_compare]) {
|
||||
// download new video as the simulated one is better
|
||||
youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
|
||||
if (err) {
|
||||
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
|
||||
} else if (output) {
|
||||
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
|
||||
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const info = await downloader_api.getVideoInfoByURL(file_obj['url'], downloadConfig);
|
||||
if (info && info.length === 1) {
|
||||
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
|
||||
if (info[metric_to_compare] > file_obj[metric_to_compare]) {
|
||||
// download new video as the simulated one is better
|
||||
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
||||
const {parsed_output, err} = await callback;
|
||||
if (err) {
|
||||
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
|
||||
} else if (parsed_output) {
|
||||
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${info[metric_to_compare]}`);
|
||||
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: info[metric_to_compare]});
|
||||
}
|
||||
}
|
||||
}
|
||||
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user