Fixed bug where if a subscription name was missing, the wrong folder was used

Fixed bug that caused high CPU usage when the subscriptions check took longer than the interval in the settings (thus they piled on)

backend/app.js

@@ -165,7 +165,9 @@ var validDownloadingAgents = [
     'ffmpeg',
     'httpie',
     'wget'
-]
+];
+
+const subscription_timeouts = {};
 
 // don't overwrite config if it already happened.. NOT
 // let alreadyWritten = db.get('configWriteFlag').value();
@@ -616,15 +618,14 @@ function loadConfigValues() {
     logger.transports[2].level = logger_level;
 }
 
-function calculateSubcriptionRetrievalDelay(amount) {
+function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
-    // frequency is 5 mins
+    // frequency is once every 5 mins by default
-    let frequency_in_ms = subscriptionsCheckInterval * 1000;
+    let interval_in_ms = subscriptionsCheckInterval * 1000;
-    let minimum_frequency = 60 * 1000;
+    const subinterval_in_ms = interval_in_ms/subscriptions_amount;
-    const first_frequency = frequency_in_ms/amount;
+    return subinterval_in_ms;
-    return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency;
 }
 
-function watchSubscriptions() { 
+async function watchSubscriptions() { 
     let subscriptions = null;
     
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
@@ -646,10 +647,19 @@ function watchSubscriptions() {
     let current_delay = 0;
     for (let i = 0; i < subscriptions.length; i++) {
         let sub = subscriptions[i];
+
+        // don't check the sub if the last check for the same subscription has not completed
+        if (subscription_timeouts[sub.id]) {
+            logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`);
+            continue;
+        }
+
         logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
-        setTimeout(() => {
+        setTimeout(async () => {
-            subscriptions_api.getVideosForSub(sub, sub.user_uid);
+            await subscriptions_api.getVideosForSub(sub, sub.user_uid);
+            subscription_timeouts[sub.id] = false;
         }, current_delay);
+        subscription_timeouts[sub.id] = true;
         current_delay += delay_interval;
         if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
     }
@@ -2212,7 +2222,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
         else
             base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
         
-        let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/');
+        let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
         let files;
         try {
             files = recFindByExt(appended_base_path, 'mp4');

backend/subscriptions.js

@@ -240,10 +240,10 @@ async function getVideosForSub(sub, user_uid = null) {
         if (sub.name) {
             appendedBasePath = getAppendedBasePath(sub, basePath);
         } else {
-            appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s');
+            appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'));
         }
 
-        let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json'];
+        let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json'];
 
         let archive_dir = null;
         let archive_path = null;
@@ -275,10 +275,28 @@ async function getVideosForSub(sub, user_uid = null) {
         }
 
         // get videos 
-        logger.verbose('Subscribe: getting videos for subscription ' + sub.name);
+        logger.verbose('Subscription: getting videos for subscription ' + sub.name);
         youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
-            if (err) {
+            logger.verbose('Subscription: finished check for ' + sub.name);
+            if (err && !output) {
                 logger.error(err.stderr);
+                if (err.stderr.includes('This video is unavailable')) {
+                    logger.info('An error was encountered with at least one video, backup method will be used.')
+                    try {
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        for (let i = 0; i < outputs.length; i++) {
+                            const output = JSON.parse(outputs[i]);
+                            handleOutputJSON(sub, sub_db, output, i === 0)
+                            if (err.stderr.includes(output['id']) && archive_path) {
+                                // we found a video that errored! add it to the archive to prevent future errors
+                                fs.appendFileSync(archive_path, output['id']);
+                            }
+                        }
+                    } catch(e) {
+                        logger.error('Backup method failed. See error below:');
+                        logger.error(e);
+                    }
+                }
                 resolve(false);
             } else if (output) {
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
@@ -296,17 +314,8 @@ async function getVideosForSub(sub, user_uid = null) {
                         continue;
                     }
 
-                    if (sub.streamingOnly) {
+                    const reset_videos = i === 0;
-                        if (i === 0) {
+                    handleOutputJSON(sub, sub_db, output_json, reset_videos);
-                            sub_db.assign({videos: []}).write();
-                        }
-
-                        // remove unnecessary info
-                        output_json.formats = null;
-
-                        // add to db
-                        sub_db.get('videos').push(output_json).write();
-                    }
 
                     // TODO: Potentially store downloaded files in db?
         
@@ -317,6 +326,20 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) {
+    if (sub.streamingOnly) {
+        if (reset_videos) {
+            sub_db.assign({videos: []}).write();
+        }
+
+        // remove unnecessary info
+        output_json.formats = null;
+
+        // add to db
+        sub_db.get('videos').push(output_json).write();
+    }
+}
+
 function getAllSubscriptions(user_uid = null) {
     if (user_uid)
         return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();