Subscription metadata is now backed up

Rebuild database now used subscription metadata backup

backend/app.js

@@ -535,6 +535,7 @@ async function loadConfig() {
     if (allowSubscriptions) {
         // set downloading to false
         let subscriptions = await subscriptions_api.getAllSubscriptions();
+        subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
         subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
         // runs initially, then runs every ${subscriptionCheckInterval} seconds
         const watchSubscriptionsInterval = function() {

backend/authentication/auth.js

@@ -1,5 +1,5 @@
 const config_api = require('../config');
-const consts = require('../consts');
+const CONSTS = require('../consts');
 const logger = require('../logger');
 const db_api = require('../db');
 
@@ -68,14 +68,7 @@ exports.initialize = function () {
 const setupRoles = async () => {
   const required_roles = {
     admin: {
-        permissions: [
-            'filemanager',
-            'settings',
-            'subscriptions',
-            'sharing',
-            'advanced_download',
-            'downloads_manager'
-        ]
+        permissions: CONSTS.AVAILABLE_PERMISSIONS
     },
     user: {
         permissions: [

backend/consts.js

@@ -347,6 +347,8 @@ const YTDL_ARGS_WITH_VALUES = [
     '--convert-subs'
 ];
 
+exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
+
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 

backend/subscriptions.js

@@ -6,13 +6,14 @@ const config_api = require('./config');
 const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
+const CONSTS = require('./consts');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 const db_api = require('./db');
 const downloader_api = require('./downloader');
 
-exports.subscribe = async (sub, user_uid = null) => {
+exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
     const result_obj = {
         success: false,
         error: ''
@@ -34,7 +35,8 @@ exports.subscribe = async (sub, user_uid = null) => {
         sub['user_uid'] = user_uid ? user_uid : undefined;
         await db_api.insertRecordIntoTable('subscriptions', sub);
 
-        let success = await getSubscriptionInfo(sub);
+        let success = skip_get_info ? true : await getSubscriptionInfo(sub);
+        exports.writeSubscriptionMetadata(sub);
 
         if (success) {
             if (!sub.paused) exports.getVideosForSub(sub, user_uid);
@@ -465,6 +467,7 @@ exports.getSubscriptionByName = async (subName, user_uid = null) => {
 
 exports.updateSubscription = async (sub) => {
     await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
+    exports.writeSubscriptionMetadata(sub);
     return true;
 }
 
@@ -480,6 +483,14 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
+exports.writeSubscriptionMetadata = (sub) => {
+    let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
+                                : config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const appendedBasePath = getAppendedBasePath(sub, basePath);
+    const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
+    fs.writeJSONSync(metadata_path, sub);
+}
+
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     if (!sub_files) return;

backend/tasks.js

@@ -8,6 +8,7 @@ const config_api = require('./config');
 const auth_api = require('./authentication/auth');
 const utils = require('./utils');
 const logger = require('./logger');
+const CONSTS = require('./consts');
 
 const fs = require('fs-extra');
 const path = require('path');
@@ -279,6 +280,7 @@ async function autoDeleteFiles(data) {
 async function rebuildDB() {
     await db_api.backupDB();
     let subs_to_add = await guessSubscriptions(false);
+    let subs_to_add = await guessSubscriptions(true);
     subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
     const users_to_add = await guessUsers();
     for (const user_to_add of users_to_add) {
@@ -298,17 +300,9 @@ async function rebuildDB() {
     for (const sub_to_add of subs_to_add) {
         const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
         // TODO: we shouldn't be creating this here
-        const new_sub = {
-            name: sub_to_add['name'],
-            url: sub_to_add['url'],
-            maxQuality: 'best',
-            id: uuid(),
-            user_uid: sub_to_add['user_uid'],
-            type: sub_to_add['type'],
-            paused: true
-        };
+        const new_sub = Object.assign({}, sub_to_add, {paused: true});
         if (!sub_exists) {
-            await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid']);
+            await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
             logger.info(`Regenerated subscription ${sub_to_add['name']}`);
         }
     }
@@ -323,7 +317,7 @@ const guessUsers = async () => {
     return userPaths.map(userPath => path.basename(userPath));
 }
 
-const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null) => {
+const guessSubscriptions = async (isPlaylist, basePath = null) => {
     const guessed_subs = [];
     const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
@@ -332,21 +326,17 @@ const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null)
 
     const subs = await utils.getDirectoriesInDirectory(subsPath);
     for (const subPath of subs) {
-        const audio_files = await utils.getDownloadedFilesByType(subPath, 'audio', true);
-        const video_files = await utils.getDownloadedFilesByType(subPath, 'video', true);
-        const files = audio_files.concat(video_files);
-    
-        if (files.length === 0) continue;
+        const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
+        if (!fs.existsSync(sub_backup_path)) continue;
 
-        const sample_file = files[0];
-        const url = sample_file['channel_url'];
-        guessed_subs.push({
-            url: url,
-            name: path.basename(subPath),
-            user_uid: user_uid,
-            type: video_files.length !== 0 ? 'video' : 'audio',
-            isPlaylist: isPlaylist
-        });
+        try {
+            const sub_backup = fs.readJSONSync(sub_backup_path)
+            delete sub_backup['_id'];
+            guessed_subs.push(sub_backup);
+        } catch(err) {
+            logger.warn(`Failed to reimport subscription in path ${subPath}`)
+            logger.warn(err);
+        }
     }
 
     return guessed_subs;