diff --git a/backend/app.js b/backend/app.js index d7b6a68..8177f20 100644 --- a/backend/app.js +++ b/backend/app.js @@ -165,7 +165,9 @@ var validDownloadingAgents = [ 'ffmpeg', 'httpie', 'wget' -] +]; + +const subscription_timeouts = {}; // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); @@ -616,15 +618,14 @@ function loadConfigValues() { logger.transports[2].level = logger_level; } -function calculateSubcriptionRetrievalDelay(amount) { - // frequency is 5 mins - let frequency_in_ms = subscriptionsCheckInterval * 1000; - let minimum_frequency = 60 * 1000; - const first_frequency = frequency_in_ms/amount; - return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency; +function calculateSubcriptionRetrievalDelay(subscriptions_amount) { + // frequency is once every 5 mins by default + let interval_in_ms = subscriptionsCheckInterval * 1000; + const subinterval_in_ms = interval_in_ms/subscriptions_amount; + return subinterval_in_ms; } -function watchSubscriptions() { +async function watchSubscriptions() { let subscriptions = null; const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); @@ -646,10 +647,19 @@ function watchSubscriptions() { let current_delay = 0; for (let i = 0; i < subscriptions.length; i++) { let sub = subscriptions[i]; + + // don't check the sub if the last check for the same subscription has not completed + if (subscription_timeouts[sub.id]) { + logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`); + continue; + } + logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval); - setTimeout(() => { - subscriptions_api.getVideosForSub(sub, sub.user_uid); + setTimeout(async () => { + await subscriptions_api.getVideosForSub(sub, sub.user_uid); + subscription_timeouts[sub.id] = false; }, current_delay); + subscription_timeouts[sub.id] = true; current_delay += delay_interval; if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; } @@ -2212,7 +2222,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { else base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); - let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/'); + let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); let files; try { files = recFindByExt(appended_base_path, 'mp4'); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index e5ba918..811c771 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -240,10 +240,10 @@ async function getVideosForSub(sub, user_uid = null) { if (sub.name) { appendedBasePath = getAppendedBasePath(sub, basePath); } else { - appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'); + appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s')); } - let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json']; + let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json']; let archive_dir = null; let archive_path = null; @@ -275,10 +275,28 @@ async function getVideosForSub(sub, user_uid = null) { } // get videos - logger.verbose('Subscribe: getting videos for subscription ' + sub.name); + logger.verbose('Subscription: getting videos for subscription ' + sub.name); youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { - if (err) { + logger.verbose('Subscription: finished check for ' + sub.name); + if (err && !output) { logger.error(err.stderr); + if (err.stderr.includes('This video is unavailable')) { + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + const outputs = err.stdout.split(/\r\n|\r|\n/); + for (let i = 0; i < outputs.length; i++) { + const output = JSON.parse(outputs[i]); + handleOutputJSON(sub, sub_db, output, i === 0) + if (err.stderr.includes(output['id']) && archive_path) { + // we found a video that errored! add it to the archive to prevent future errors + fs.appendFileSync(archive_path, output['id']); + } + } + } catch(e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + } + } resolve(false); } else if (output) { if (output.length === 0 || (output.length === 1 && output[0] === '')) { @@ -296,17 +314,8 @@ async function getVideosForSub(sub, user_uid = null) { continue; } - if (sub.streamingOnly) { - if (i === 0) { - sub_db.assign({videos: []}).write(); - } - - // remove unnecessary info - output_json.formats = null; - - // add to db - sub_db.get('videos').push(output_json).write(); - } + const reset_videos = i === 0; + handleOutputJSON(sub, sub_db, output_json, reset_videos); // TODO: Potentially store downloaded files in db? @@ -317,6 +326,20 @@ async function getVideosForSub(sub, user_uid = null) { }); } +function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) { + if (sub.streamingOnly) { + if (reset_videos) { + sub_db.assign({videos: []}).write(); + } + + // remove unnecessary info + output_json.formats = null; + + // add to db + sub_db.get('videos').push(output_json).write(); + } +} + function getAllSubscriptions(user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();