Adds task to rebuild database

backend/subscriptions.js

@@ -19,7 +19,7 @@ async function subscribe(sub, user_uid = null) {
     };
     return new Promise(async resolve => {
         // sub should just have url and name. here we will get isPlaylist and path
-        sub.isPlaylist = sub.url.includes('playlist');
+        sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
         sub.videos = [];
 
         let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -37,7 +37,7 @@ async function subscribe(sub, user_uid = null) {
         let success = await getSubscriptionInfo(sub);
 
         if (success) {
-            getVideosForSub(sub, user_uid);
+            if (!sub.paused) getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
         }
@@ -454,7 +454,7 @@ async function getSubscription(subID) {
     // stringify and parse because we may override the 'downloading' property
     const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
     // now with the download_queue, we may need to override 'downloading'
-    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
     if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
     return sub;
 }

backend/tasks.js

@@ -3,10 +3,16 @@ const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
 const archive_api = require('./archive');
 const files_api = require('./files');
+const subscriptions_api = require('./subscriptions');
+const config_api = require('./config');
+const auth_api = require('./authentication/auth');
+const utils = require('./utils');
+const logger = require('./logger');
 
 const fs = require('fs-extra');
-const logger = require('./logger');
+const path = require('path');
 const scheduler = require('node-schedule');
+const { uuid } = require('uuidv4');
 
 const TASKS = {
     backup_local_db: {
@@ -47,6 +53,11 @@ const TASKS = {
         run: archive_api.importArchives,
         title: 'Import legacy archives',
         job: null
+    },
+    rebuild_database: {
+        run: rebuildDB,
+        title: 'Rebuild database',
+        job: null
     }
 }
 
@@ -265,4 +276,73 @@ async function autoDeleteFiles(data) {
     }
 }
 
+async function rebuildDB() {
+    await db_api.backupDB();
+    let subs_to_add = await guessSubscriptions(false);
+    subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
+    const users_to_add = await guessUsers();
+    for (const user_to_add of users_to_add) {
+        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        
+        const user_exists = await db_api.getRecord('users', {uid: user_to_add});
+        if (!user_exists) await auth_api.registerUser(user_to_add, user_to_add, '');
+        else logger.info(`Regenerated user ${user_to_add}`);
+        
+        const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
+        const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
+        subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
+    }
+
+    for (const sub_to_add of subs_to_add) {
+        const sub_exists = await db_api.getRecord('subscriptions', {name: sub_to_add['name']});
+        // TODO: we shouldn't be creating this here
+        const new_sub = {
+            name: sub_to_add['name'],
+            url: sub_to_add['url'],
+            maxQuality: 'best',
+            id: uuid(),
+            user_uid: sub_to_add['user_uid'],
+            type: sub_to_add['type'],
+            paused: true
+        };
+        if (!sub_exists) await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid']);
+        else logger.info(`Regenerated subscription ${sub_to_add['name']}`);
+    }
+}
+
+const guessUsers = async () => {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
+    return userPaths.map(userPath => path.basename(userPath));
+}
+
+const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null) => {
+    const guessed_subs = [];
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
+    const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
+
+    const subs = await utils.getDirectoriesInDirectory(subsPath);
+    for (const subPath of subs) {
+        const audio_files = await utils.getDownloadedFilesByType(subPath, 'audio', true);
+        const video_files = await utils.getDownloadedFilesByType(subPath, 'video', true);
+        const files = audio_files.concat(video_files);
+    
+        if (files.length === 0) continue;
+
+        const sample_file = files[0];
+        const url = sample_file['channel_url'];
+        guessed_subs.push({
+            url: url,
+            name: path.basename(subPath),
+            user_uid: user_uid,
+            type: video_files.length !== 0 ? 'video' : 'audio',
+            isPlaylist: isPlaylist
+        });
+    }
+
+    return guessed_subs;
+}
+
 exports.TASKS = TASKS;

backend/utils.js

@@ -519,6 +519,17 @@ exports.convertFlatObjectToNestedObject = (obj) => {
     return result;
 }
 
+exports.getDirectoriesInDirectory = async (basePath) => {
+    try {
+        const files = await fs.readdir(basePath, { withFileTypes: true });
+        return files
+            .filter((file) => file.isDirectory())
+            .map((file) => path.join(basePath, file.name));
+    } catch (err) {
+        return [];
+    }
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {