Merge pull request #195 from Tzahi12345/import-files-on-startup

Import files on startup

backend/app.js

@@ -585,6 +585,8 @@ async function loadConfig() {
             }, subscriptionsCheckInterval * 1000);
         }
 
+        db_api.importUnregisteredFiles();
+
         // check migrations
         await checkMigrations();
 
@@ -730,7 +732,7 @@ function generateEnvVarConfigItem(key) {
 
 function getMp3s() {
     let mp3s = [];
-    var files = recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
+    var files = utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
     for (let i = 0; i < files.length; i++) {
         let file = files[i];
         var file_path = file.substring(audioFolderPath.length, file.length);
@@ -759,7 +761,7 @@ function getMp3s() {
 
 function getMp4s(relative_path = true) {
     let mp4s = [];
-    var files = recFindByExt(videoFolderPath, 'mp4');
+    var files = utils.recFindByExt(videoFolderPath, 'mp4');
     for (let i = 0; i < files.length; i++) {
         let file = files[i];
         var file_path = file.substring(videoFolderPath.length, file.length);
@@ -1078,88 +1080,6 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
     });
 }
 
-function recFindByExt(base,ext,files,result)
-{
-    files = files || fs.readdirSync(base)
-    result = result || []
-
-    files.forEach(
-        function (file) {
-            var newbase = path.join(base,file)
-            if ( fs.statSync(newbase).isDirectory() )
-            {
-                result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
-            }
-            else
-            {
-                if ( file.substr(-1*(ext.length+1)) == '.' + ext )
-                {
-                    result.push(newbase)
-                }
-            }
-        }
-    )
-    return result
-}
-/*
-function registerFileDB(file_path, type, multiUserMode = null) {
-    const file_id = file_path.substring(0, file_path.length-4);
-    const file_object = generateFileObject(file_id, type, multiUserMode && multiUserMode.file_path);
-    if (!file_object) {
-        logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
-        return false;
-    }
-
-    // add additional info
-    file_object['uid'] = uuid();
-    file_object['registered'] = Date.now();
-    path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
-
-    if (multiUserMode) {
-        auth_api.registerUserFile(multiUserMode.user, file_object, type);
-    } else if (type === 'audio' || type === 'video') {
-        // remove existing video if overwriting
-        db.get(`files.${type}`)
-        .remove({
-            path: file_object['path']
-        }).write();
-
-        db.get(`files.${type}`)
-            .push(file_object)
-            .write();
-    } else if (type == 'subscription') {
-
-    }
-
-    return file_object['uid'];
-}
-
-function generateFileObject(id, type, customPath = null) {
-    var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
-    if (!jsonobj) {
-        return null;
-    }
-    const ext = (type === 'audio') ? '.mp3' : '.mp4'
-    const file_path = getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
-    // console.
-    var stats = fs.statSync(path.join(__dirname, file_path));
-
-    var title = jsonobj.title;
-    var url = jsonobj.webpage_url;
-    var uploader = jsonobj.uploader;
-    var upload_date = jsonobj.upload_date;
-    upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
-
-    var size = stats.size;
-
-    var thumbnail = jsonobj.thumbnail;
-    var duration = jsonobj.duration;
-    var isaudio = type === 'audio';
-    var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date);
-    return file_obj;
-}
-*/
 // replaces .webm with appropriate extension
 function getTrueFileName(unfixed_path, type) {
     let fixed_path = unfixed_path;
@@ -2292,7 +2212,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
             let files;
             try {
-                files = recFindByExt(appended_base_path, 'mp4');
+                files = utils.recFindByExt(appended_base_path, 'mp4');
             } catch(e) {
                 files = null;
                 logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);

backend/db.js

@@ -116,8 +116,89 @@ function getAppendedBasePathSub(sub, base_path) {
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
+async function importUnregisteredFiles() {
+    let dirs_to_check = [];
+    let subscriptions_to_check = [];
+    const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode
+    const multi_user_mode = config_api.getConfigItem('ytdl_multi_user_mode');
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions');
+    if (multi_user_mode) {
+        let users = users_db.get('users').value();
+        for (let i = 0; i < users.length; i++) {
+            const user = users[i];
+
+            if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']);
+
+            // add user's audio dir to check list
+            dirs_to_check.push({
+                basePath: path.join(usersFileFolder, user.uid, 'audio'),
+                dbPath: users_db.get('users').find({uid: user.uid}).get('files.audio'),
+                type: 'audio'
+            });
+
+            // add user's video dir to check list
+            dirs_to_check.push({
+                basePath: path.join(usersFileFolder, user.uid, 'video'),
+                dbPath: users_db.get('users').find({uid: user.uid}).get('files.video'),
+                type: 'video'
+            });
+        }
+    } else {
+        const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+        const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const subscriptions = db.get('subscriptions').value();
+
+        if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
+
+        // add audio dir to check list
+        dirs_to_check.push({
+            basePath: audioFolderPath,
+            dbPath: db.get('files.audio'),
+            type: 'audio'
+        });
+
+        // add video dir to check list
+        dirs_to_check.push({
+            basePath: videoFolderPath,
+            dbPath: db.get('files.video'),
+            type: 'video'
+        });
+    }
+
+    // add subscriptions to check list
+    for (let i = 0; i < subscriptions_to_check.length; i++) {
+        let subscription_to_check = subscriptions_to_check[i];
+        console.log(subscription_to_check);
+        dirs_to_check.push({
+            basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
+                                      : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
+            dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos')
+                                    : db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'),
+            type: subscription_to_check.type
+        });
+    }
+
+    // run through check list and check each file to see if it's missing from the db
+    dirs_to_check.forEach(dir_to_check => {
+        // recursively get all files in dir's path
+        const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
+
+        files.forEach(file => {
+            // check if file exists in db, if not add it
+            const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value())
+            if (!file_is_registered) {
+                dir_to_check.dbPath.push(file).write();
+                logger.verbose(`Added discovered file to the database: ${file.id}`);
+            }
+        });
+    });
+
+}
+
 module.exports = {
     initialize: initialize,
     registerFileDB: registerFileDB,
-    updatePlaylist: updatePlaylist
+    updatePlaylist: updatePlaylist,
+    importUnregisteredFiles: importUnregisteredFiles
 }

backend/utils.js

@@ -19,6 +19,36 @@ function getTrueFileName(unfixed_path, type) {
     return fixed_path;
 }
 
+function getDownloadedFilesByType(basePath, type) {
+    let files = [];
+    const ext = type === 'audio' ? 'mp3' : 'mp4';
+    var located_files = recFindByExt(basePath, ext);
+    for (let i = 0; i < located_files.length; i++) {
+        let file = located_files[i];
+        var file_path = path.basename(file);
+
+        var stats = fs.statSync(file);
+
+        var id = file_path.substring(0, file_path.length-4);
+        var jsonobj = getJSONByType(type, id, basePath);
+        if (!jsonobj) continue;
+        var title = jsonobj.title;
+        var url = jsonobj.webpage_url;
+        var uploader = jsonobj.uploader;
+        var upload_date = jsonobj.upload_date;
+        upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
+        var thumbnail = jsonobj.thumbnail;
+        var duration = jsonobj.duration;
+
+        var size = stats.size;
+
+        var isaudio = type === 'audio';
+        var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
+        files.push(file_obj);
+    }
+    return files;
+}
+
 function getJSONMp4(name, customPath, openReadPerms = false) {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -51,6 +81,10 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
+function getJSONByType(type, name, customPath, openReadPerms = false) {
+    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+}
+
 function fixVideoMetadataPerms(name, type, customPath = null) {
     if (is_windows) return;
     if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
@@ -73,6 +107,30 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
     }
 }
 
+function recFindByExt(base,ext,files,result)
+{
+    files = files || fs.readdirSync(base)
+    result = result || []
+
+    files.forEach(
+        function (file) {
+            var newbase = path.join(base,file)
+            if ( fs.statSync(newbase).isDirectory() )
+            {
+                result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
+            }
+            else
+            {
+                if ( file.substr(-1*(ext.length+1)) == '.' + ext )
+                {
+                    result.push(newbase)
+                }
+            }
+        }
+    )
+    return result
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) {
@@ -93,5 +151,7 @@ module.exports = {
     getJSONMp4: getJSONMp4,
     getTrueFileName: getTrueFileName,
     fixVideoMetadataPerms: fixVideoMetadataPerms,
+    getDownloadedFilesByType: getDownloadedFilesByType,
+    recFindByExt: recFindByExt,
     File: File
 }