Added support for redownloading fresh uploads, which will eventually be hidden behind an opt-in setting

backend/app.js

@@ -218,6 +218,8 @@ async function checkMigrations() {
         let success = await simplifyDBFileStructure();
         success = success && await addMetadataPropertyToDB('view_count');
         success = success && await addMetadataPropertyToDB('description');
+        success = success && await addMetadataPropertyToDB('height');
+        success = success && await addMetadataPropertyToDB('abr');
         if (success) { logger.info('4.1->4.2+ migration complete!'); }
         else { logger.error('Migration failed: 4.1->4.2+'); }
     }
@@ -225,7 +227,6 @@ async function checkMigrations() {
     return true;
 }
 
-/*
 async function runFilesToDBMigration() {
     try {
         let mp3s = await getMp3s();
@@ -257,7 +258,6 @@ async function runFilesToDBMigration() {
         return false;
     }
 }
-*/
 
 async function simplifyDBFileStructure() {
     let users = users_db.get('users').value();
@@ -760,64 +760,6 @@ function generateEnvVarConfigItem(key) {
     return {key: key, value: process['env'][key]};
 }
 
-async function getMp3s() {
-    let mp3s = [];
-    var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
-    for (let i = 0; i < files.length; i++) {
-        let file = files[i];
-        var file_path = file.substring(audioFolderPath.length, file.length);
-
-        var stats = await fs.stat(file);
-
-        var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await utils.getJSONMp3(id, audioFolderPath);
-        if (!jsonobj) continue;
-        var title = jsonobj.title;
-        var url = jsonobj.webpage_url;
-        var uploader = jsonobj.uploader;
-        var upload_date = jsonobj.upload_date;
-        upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
-
-        var size = stats.size;
-
-        var thumbnail = jsonobj.thumbnail;
-        var duration = jsonobj.duration;
-        var isaudio = true;
-        var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
-        mp3s.push(file_obj);
-    }
-    return mp3s;
-}
-
-async function getMp4s(relative_path = true) {
-    let mp4s = [];
-    var files = await utils.recFindByExt(videoFolderPath, 'mp4');
-    for (let i = 0; i < files.length; i++) {
-        let file = files[i];
-        var file_path = file.substring(videoFolderPath.length, file.length);
-
-        var stats = fs.statSync(file);
-
-        var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await utils.getJSONMp4(id, videoFolderPath);
-        if (!jsonobj) continue;
-        var title = jsonobj.title;
-        var url = jsonobj.webpage_url;
-        var uploader = jsonobj.uploader;
-        var upload_date = jsonobj.upload_date;
-        upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
-        var thumbnail = jsonobj.thumbnail;
-        var duration = jsonobj.duration;
-
-        var size = stats.size;
-
-        var isaudio = false;
-        var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
-        mp4s.push(file_obj);
-    }
-    return mp4s;
-}
-
 function getThumbnailMp3(name)
 {
     var obj = utils.getJSONMp3(name, audioFolderPath);
@@ -2446,7 +2388,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
                 var size = stats.size;
 
                 var isaudio = false;
-                var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
+                var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
                 parsed_files.push(file_obj);
             }
         } else {
@@ -2468,7 +2410,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
         if (subscription.videos) {
             for (let i = 0; i < subscription.videos.length; i++) {
                 const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date));
+                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
             }
         }
         res.send({

backend/db.js

@@ -95,7 +95,7 @@ function generateFileObject(id, type, customPath = null, sub = null) {
     var duration = jsonobj.duration;
     var isaudio = type === 'audio';
     var description = jsonobj.description;
-    var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description);
+    var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
     return file_obj;
 }
 

backend/subscriptions.js

@@ -273,10 +273,7 @@ async function getVideosForSub(sub, user_uid = null) {
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    let appendedBasePath = getAppendedBasePath(sub, basePath);
-
-    let appendedBasePath = null
-    appendedBasePath = getAppendedBasePath(sub, basePath);
 
     let multiUserMode = null;
     if (user_uid) {
@@ -286,14 +283,87 @@ async function getVideosForSub(sub, user_uid = null) {
         }
     }
 
-    const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
+    const downloadConfig = await generateArgsForSubscription(sub, user_uid);
+
+    // get videos
+    logger.verbose('Subscription: getting videos for subscription ' + sub.name);
+
+    return new Promise(resolve => {
+        youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) {
+            logger.verbose('Subscription: finished check for ' + sub.name);
+            if (err && !output) {
+                logger.error(err.stderr ? err.stderr : err.message);
+                if (err.stderr.includes('This video is unavailable')) {
+                    logger.info('An error was encountered with at least one video, backup method will be used.')
+                    try {
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        for (let i = 0; i < outputs.length; i++) {
+                            const output = JSON.parse(outputs[i]);
+                            handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
+                            if (err.stderr.includes(output['id']) && archive_path) {
+                                // we found a video that errored! add it to the archive to prevent future errors
+                                if (sub.archive) {
+                                    archive_dir = sub.archive;
+                                    archive_path = path.join(archive_dir, 'archive.txt')
+                                    fs.appendFileSync(archive_path, output['id']);
+                                }
+                            }
+                        }
+                    } catch(e) {
+                        logger.error('Backup method failed. See error below:');
+                        logger.error(e);
+                    }
+                }
+                resolve(false);
+            } else if (output) {
+                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+                    logger.verbose('No additional videos to download for ' + sub.name);
+                    resolve(true);
+                }
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+                    if (!output_json) {
+                        continue;
+                    }
+
+                    const reset_videos = i === 0;
+                    handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
+                }
+                resolve(true);
+            }
+        });
+    }, err => {
+        logger.error(err);
+    });
+}
+
+async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
+    // get basePath
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+
+    let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
-    if (sub.custom_output) {
+    if (desired_path) {
+        fullOutput = `${desired_path}.%(ext)s`;
+    } else if (sub.custom_output) {
         fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
     }
 
-    let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
+    let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
 
     let qualityPath = null;
     if (sub.type && sub.type === 'audio') {
@@ -320,7 +390,7 @@ async function getVideosForSub(sub, user_uid = null) {
     let archive_dir = null;
     let archive_path = null;
 
-    if (useArchive) {
+    if (useArchive && !redownload) {
         if (sub.archive) {
             archive_dir = sub.archive;
             archive_path = path.join(archive_dir, 'archive.txt')
@@ -350,60 +420,7 @@ async function getVideosForSub(sub, user_uid = null) {
         downloadConfig.push('--write-thumbnail');
     }
 
-    // get videos
+    return downloadConfig;
-    logger.verbose('Subscription: getting videos for subscription ' + sub.name);
-
-    return new Promise(resolve => {
-        youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
-            logger.verbose('Subscription: finished check for ' + sub.name);
-            if (err && !output) {
-                logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable')) {
-                    logger.info('An error was encountered with at least one video, backup method will be used.')
-                    try {
-                        const outputs = err.stdout.split(/\r\n|\r|\n/);
-                        for (let i = 0; i < outputs.length; i++) {
-                            const output = JSON.parse(outputs[i]);
-                            handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
-                            if (err.stderr.includes(output['id']) && archive_path) {
-                                // we found a video that errored! add it to the archive to prevent future errors
-                                fs.appendFileSync(archive_path, output['id']);
-                            }
-                        }
-                    } catch(e) {
-                        logger.error('Backup method failed. See error below:');
-                        logger.error(e);
-                    }
-                }
-                resolve(false);
-            } else if (output) {
-                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
-                    logger.verbose('No additional videos to download for ' + sub.name);
-                    resolve(true);
-                }
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                    } catch(e) {
-                        output_json = null;
-                    }
-                    if (!output_json) {
-                        continue;
-                    }
-
-                    const reset_videos = i === 0;
-                    handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
-
-                    // TODO: Potentially store downloaded files in db?
-
-                }
-                resolve(true);
-            }
-        });
-    }, err => {
-        logger.error(err);
-    });
 }
 
 function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
@@ -418,6 +435,14 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
         // add to db
         sub_db.get('videos').push(output_json).write();
     } else {
+        path_object = path.parse(output_json['_filename']);
+        const path_string = path.format(path_object);
+
+        if (sub_db.get('videos').find({path: path_string}).value()) {
+            // file already exists in DB, return early to avoid reseting the download date
+            return;
+        }
+
         db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
         const url = output_json['webpage_url'];
         if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
@@ -468,6 +493,53 @@ function subExists(subID, user_uid = null) {
         return !!db.get('subscriptions').find({id: subID}).value();
 }
 
+async function setFreshUploads(sub, user_uid) {
+    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
+    sub.videos.forEach(async video => {
+        if (current_date === video['upload_date'].replace(/-/g, '')) {
+            // set upload as fresh
+            const video_uid = video['uid'];
+            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+        }
+    });
+}
+
+async function checkVideosForFreshUploads(sub, user_uid) {
+    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
+    sub.videos.forEach(async video => {
+        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
+            checkVideoIfBetterExists(video, sub, user_uid)
+        }
+    });
+}
+
+async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
+    const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
+    const downloadConfig = generateArgsForSubscription(sub, user_uid, true, new_path);
+    logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
+    // simulate a download to verify that a better version exists
+    youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
+        if (err) {
+            // video is not available anymore for whatever reason
+        } else if (output) {
+            console.log(output);
+            const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
+            if (output[metric_to_compare] > file_obj[metric_to_compare]) {
+                // download new video as the simulated one is better
+                youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => {
+                    if (err) {
+                        logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
+                    } else if (output) {
+                        logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                    }
+                });
+            } 
+        }
+    });
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+}
+
 // helper functions
 
 function getAppendedBasePath(sub, base_path) {

backend/utils.js

@@ -41,18 +41,12 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
             files.push(jsonobj);
             continue;
         }
-        var title = jsonobj.title;
-        var url = jsonobj.webpage_url;
-        var uploader = jsonobj.uploader;
         var upload_date = jsonobj.upload_date;
         upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
-        var thumbnail = jsonobj.thumbnail;
-        var duration = jsonobj.duration;
-
-        var size = stats.size;
 
         var isaudio = type === 'audio';
-        var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
+        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+                                stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
@@ -189,7 +183,7 @@ async function recFindByExt(base,ext,files,result)
 
 // objects
 
-function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description) {
+function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
     this.id = id;
     this.title = title;
     this.thumbnailURL = thumbnailURL;
@@ -201,6 +195,9 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.path = path;
     this.upload_date = upload_date;
     this.description = description;
+    this.view_count = view_count;
+    this.height = height;
+    this.abr = abr;
 }
 
 module.exports = {