TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 20:10:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed bug where if a subscription name was missing, the wrong folder was used

Browse Source 
Fixed bug that caused high CPU usage when the subscriptions check took longer than the interval in the settings (thus they piled on)
This commit is contained in:
Isaac Grynsztein
2020-05-25 15:35:49 -04:00
parent 73cd142b77
commit c8a8046056
2 changed files with 59 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
backend/app.js
View File

@@ -165,7 +165,9 @@ var validDownloadingAgents = [
'ffmpeg',
'httpie',
'wget'
]
];
const subscription_timeouts = {};
// don't overwrite config if it already happened.. NOT
// let alreadyWritten = db.get('configWriteFlag').value();
@@ -616,15 +618,14 @@ function loadConfigValues() {
logger.transports[2].level = logger_level;
}
function calculateSubcriptionRetrievalDelay(amount) {
// frequency is 5 mins
let frequency_in_ms = subscriptionsCheckInterval * 1000;
let minimum_frequency = 60 * 1000;
const first_frequency = frequency_in_ms/amount;
return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency;
function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
// frequency is once every 5 mins by default
let interval_in_ms = subscriptionsCheckInterval * 1000;
const subinterval_in_ms = interval_in_ms/subscriptions_amount;
return subinterval_in_ms;
}
function watchSubscriptions() {
async function watchSubscriptions() {
let subscriptions = null;
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
@@ -646,10 +647,19 @@ function watchSubscriptions() {
let current_delay = 0;
for (let i = 0; i < subscriptions.length; i++) {
let sub = subscriptions[i];
// don't check the sub if the last check for the same subscription has not completed
if (subscription_timeouts[sub.id]) {
logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`);
continue;
}
logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
setTimeout(() => {
subscriptions_api.getVideosForSub(sub, sub.user_uid);
setTimeout(async () => {
await subscriptions_api.getVideosForSub(sub, sub.user_uid);
subscription_timeouts[sub.id] = false;
}, current_delay);
subscription_timeouts[sub.id] = true;
current_delay += delay_interval;
if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
}
@@ -2212,7 +2222,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
else
base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/');
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = recFindByExt(appended_base_path, 'mp4');

53
backend/subscriptions.js
View File

@@ -240,10 +240,10 @@ async function getVideosForSub(sub, user_uid = null) {
if (sub.name) {
appendedBasePath = getAppendedBasePath(sub, basePath);
} else {
appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s');
appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'));
}
let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json'];
let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json'];
let archive_dir = null;
let archive_path = null;
@@ -275,10 +275,28 @@ async function getVideosForSub(sub, user_uid = null) {
}
// get videos
logger.verbose('Subscribe: getting videos for subscription ' + sub.name);
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
if (err) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
fs.appendFileSync(archive_path, output['id']);
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
@@ -296,17 +314,8 @@ async function getVideosForSub(sub, user_uid = null) {
continue;
}
if (sub.streamingOnly) {
if (i === 0) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, reset_videos);
// TODO: Potentially store downloaded files in db?
@@ -317,6 +326,20 @@ async function getVideosForSub(sub, user_uid = null) {
});
}
function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) {
if (sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
}
}
function getAllSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();