Fixed issue where sub was assumed as duplicate always if using local db

* Upgraded version to 4.3.2
* Prevent null userid/username registration

.github/workflows/docker.yml

@@ -80,7 +80,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
           push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}

Public API v1.yaml

@@ -1758,14 +1758,14 @@ components:
       type: object
       properties:
         task_key:
-          type: string
+          $ref: '#/components/schemas/TaskType'
       required:
         - task_key
     UpdateTaskScheduleRequest:
       type: object
       properties:
         task_key:
-          type: string
+          $ref: '#/components/schemas/TaskType'
         new_schedule:
           $ref: '#/components/schemas/Schedule'
       required:
@@ -1775,7 +1775,7 @@ components:
       type: object
       properties:
         task_key:
-          type: string
+          $ref: '#/components/schemas/TaskType'
         new_data:
           type: object
       required:
@@ -1785,7 +1785,7 @@ components:
       type: object
       properties:
         task_key:
-          type: string
+          $ref: '#/components/schemas/TaskType'
         new_options:
           type: object
       required:
@@ -2726,7 +2726,7 @@ components:
       type: object
       properties:
         key:
-          type: string
+          $ref: '#/components/schemas/TaskType'
         title:
           type: string
         last_ran:
@@ -2745,6 +2745,17 @@ components:
           $ref: '#/components/schemas/Schedule'
         options:
           type: object
+    TaskType:
+      type: string
+      enum:
+      - backup_local_db
+      - missing_files_check
+      - missing_db_records
+      - duplicate_files_check
+      - youtubedl_update_check
+      - delete_old_files
+      - import_legacy_archives
+      - rebuild_database
     Schedule:
       required:
         - type

backend/app.js

@@ -535,6 +535,7 @@ async function loadConfig() {
     if (allowSubscriptions) {
         // set downloading to false
         let subscriptions = await subscriptions_api.getAllSubscriptions();
+        subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
         subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
         // runs initially, then runs every ${subscriptionCheckInterval} seconds
         const watchSubscriptionsInterval = function() {
@@ -1928,9 +1929,38 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
 
 // user authentication
 
-app.post('/api/auth/register'
-        , optionalJwt
-        , auth_api.registerUser);
+app.post('/api/auth/register', optionalJwt, async (req, res) => {
+    const userid = req.body.userid;
+    const username = req.body.username;
+    const plaintextPassword = req.body.password;
+
+    if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
+        logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
+        res.sendStatus(409);
+        return;
+    }
+
+    if (plaintextPassword === "") {
+        logger.error(`Registration failed for user ${userid}. A password must be provided.`);
+        res.sendStatus(409);
+        return;
+    }
+
+    if (!userid || !username) {
+        logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
+    }
+  
+    const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
+  
+    if (!new_user) {
+      res.sendStatus(409);
+      return;
+    }
+  
+    res.send({
+      user: new_user
+    });
+});
 app.post('/api/auth/login'
         , auth_api.passport.authenticate(['local', 'ldapauth'], {})
         , auth_api.generateJWT
@@ -1982,18 +2012,7 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
 app.post('/api/deleteUser', optionalJwt, async (req, res) => {
     let uid = req.body.uid;
     try {
-        let success = false;
-        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const user_folder = path.join(__dirname, usersFileFolder, uid);
-        const user_db_obj = await db_api.getRecord('users', {uid: uid});
-        if (user_db_obj) {
-            // user exists, let's delete
-            await fs.remove(user_folder);
-            await db_api.removeRecord('users', {uid: uid});
-            success = true;
-        } else {
-            logger.error(`Could not find user with uid ${uid}`);
-        }
+        const success = await auth_api.deleteUser(uid);
         res.send({success: success});
     } catch (err) {
         logger.error(err);

backend/authentication/auth.js

@@ -1,11 +1,13 @@
 const config_api = require('../config');
-const consts = require('../consts');
+const CONSTS = require('../consts');
 const logger = require('../logger');
 const db_api = require('../db');
 
 const jwt = require('jsonwebtoken');
 const { uuid } = require('uuidv4');
 const bcrypt = require('bcryptjs');
+const fs = require('fs-extra');
+const path = require('path');
 
 var LocalStrategy = require('passport-local').Strategy;
 var LdapStrategy = require('passport-ldapauth');
@@ -16,7 +18,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
 let SERVER_SECRET = null;
 let JWT_EXPIRATION = null;
 let opts = null;
-let saltRounds = null;
+let saltRounds = 10;
 
 exports.initialize = function () {
   /*************************
@@ -31,8 +33,6 @@ exports.initialize = function () {
       });
   }
 
-  saltRounds = 10;
-
   // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
   if (!(+JWT_EXPIRATION)) {
@@ -68,7 +68,7 @@ exports.initialize = function () {
 const setupRoles = async () => {
   const required_roles = {
     admin: {
-        permissions: consts.AVAILABLE_PERMISSIONS
+        permissions: CONSTS.AVAILABLE_PERMISSIONS
     },
     user: {
         permissions: [
@@ -106,55 +106,41 @@ exports.passport.deserializeUser(function(user, done) {
 /***************************************
  * Register user with hashed password
  **************************************/
-exports.registerUser = async function(req, res) {
-  var userid = req.body.userid;
-  var username = req.body.username;
-  var plaintextPassword = req.body.password;
 
-  if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
-    res.sendStatus(409);
-    logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
-    return;
+exports.registerUser = async (userid, username, plaintextPassword) => {
+  const hash = await bcrypt.hash(plaintextPassword, saltRounds);
+  const new_user = generateUserObject(userid, username, hash);
+  // check if user exists
+  if (await db_api.getRecord('users', {uid: userid})) {
+    // user id is taken!
+    logger.error('Registration failed: UID is already taken!');
+    return null;
+  } else if (await db_api.getRecord('users', {name: username})) {
+      // user name is taken!
+      logger.error('Registration failed: User name is already taken!');
+      return null;
+  } else {
+    // add to db
+    await db_api.insertRecordIntoTable('users', new_user);
+    logger.verbose(`New user created: ${new_user.name}`);
+    return new_user;
   }
+}
 
-  if (plaintextPassword === "") {
-    res.sendStatus(400);
-    logger.error(`Registration failed for user ${userid}. A password must be provided.`);
-    return;
+exports.deleteUser = async (uid) => {
+  let success = false;
+  let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+  const user_folder = path.join(__dirname, usersFileFolder, uid);
+  const user_db_obj = await db_api.getRecord('users', {uid: uid});
+  if (user_db_obj) {
+      // user exists, let's delete
+      await fs.remove(user_folder);
+      await db_api.removeRecord('users', {uid: uid});
+      success = true;
+  } else {
+      logger.error(`Could not find user with uid ${uid}`);
   }
-
-  bcrypt.hash(plaintextPassword, saltRounds)
-    .then(async function(hash) {
-      let new_user = generateUserObject(userid, username, hash);
-      // check if user exists
-      if (await db_api.getRecord('users', {uid: userid})) {
-        // user id is taken!
-        logger.error('Registration failed: UID is already taken!');
-        res.status(409).send('UID is already taken!');
-      } else if (await db_api.getRecord('users', {name: username})) {
-          // user name is taken!
-          logger.error('Registration failed: User name is already taken!');
-          res.status(409).send('User name is already taken!');
-      } else {
-        // add to db
-        await db_api.insertRecordIntoTable('users', new_user);
-        logger.verbose(`New user created: ${new_user.name}`);
-        res.send({
-          user: new_user
-        });
-      }
-    })
-    .then(function(result) {
-
-    })
-    .catch(function(err) {
-      logger.error(err);
-      if( err.code == 'ER_DUP_ENTRY' ) {
-        res.status(409).send('UserId already taken');
-      } else {
-        res.sendStatus(409);
-      }
-    });
+  return success;
 }
 
 /***************************************
@@ -235,7 +221,7 @@ exports.returnAuthResponse = async function(req, res) {
     user: req.user,
     token: req.token,
     permissions: await exports.userPermissions(req.user.uid),
-    available_permissions: consts['AVAILABLE_PERMISSIONS']
+    available_permissions: CONSTS.AVAILABLE_PERMISSIONS
   });
 }
 
@@ -319,7 +305,7 @@ exports.getUserVideos = async function(user_uid, type) {
 }
 
 exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
-  let file = await db_api.getRecord('files', {file_uid: file_uid});
+  let file = await db_api.getRecord('files', {uid: file_uid});
 
   // prevent unauthorized users from accessing the file info
   if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -406,8 +392,8 @@ exports.userPermissions = async function(user_uid) {
   const role_obj = await db_api.getRecord('roles', {key: role});
   const role_permissions = role_obj['permissions'];
 
-  for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
-    let permission = consts['AVAILABLE_PERMISSIONS'][i];
+  for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
+    let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
 
     const user_has_explicit_permission = user_obj['permissions'].includes(permission);
     const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

backend/consts.js

@@ -347,6 +347,8 @@ const YTDL_ARGS_WITH_VALUES = [
     '--convert-subs'
 ];
 
+exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
+
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 

backend/entrypoint.sh

@@ -1,17 +1,7 @@
-#!/bin/sh
+#!/bin/bash
 set -eu
 
-CMD="npm start && pm2 start"
-
-# if the first arg starts with "-" pass it to program
-if [ "${1#-}" != "$1" ]; then
-  set -- "$CMD" "$@"
-fi
-
 # chown current working directory to current user
-if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
-  exec gosu "$UID:$GID" "$0" "$@"
-fi
-
-exec "$@"
+echo "[entrypoint] setup permission, this may take a while"
+find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+exec gosu "$UID:$GID" "$@"

backend/subscriptions.js

@@ -6,20 +6,21 @@ const config_api = require('./config');
 const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
+const CONSTS = require('./consts');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 const db_api = require('./db');
 const downloader_api = require('./downloader');
 
-async function subscribe(sub, user_uid = null) {
+exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
     const result_obj = {
         success: false,
         error: ''
     };
     return new Promise(async resolve => {
         // sub should just have url and name. here we will get isPlaylist and path
-        sub.isPlaylist = sub.url.includes('playlist');
+        sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
         sub.videos = [];
 
         let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -32,12 +33,13 @@ async function subscribe(sub, user_uid = null) {
         }
 
         sub['user_uid'] = user_uid ? user_uid : undefined;
-        await db_api.insertRecordIntoTable('subscriptions', sub);
+        await db_api.insertRecordIntoTable('subscriptions', JSON.parse(JSON.stringify(sub)));
 
-        let success = await getSubscriptionInfo(sub);
+        let success = skip_get_info ? true : await getSubscriptionInfo(sub);
+        exports.writeSubscriptionMetadata(sub);
 
         if (success) {
-            getVideosForSub(sub, user_uid);
+            if (!sub.paused) exports.getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
         }
@@ -109,7 +111,7 @@ async function getSubscriptionInfo(sub) {
     });
 }
 
-async function unsubscribe(sub, deleteMode, user_uid = null) {
+exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -148,7 +150,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
     await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
-async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
     if (typeof sub === 'string') {
         // TODO: fix bad workaround where sub is a sub_id
         sub = await db_api.getRecord('subscriptions', {sub_id: sub});
@@ -216,8 +218,8 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     }
 }
 
-async function getVideosForSub(sub, user_uid = null) {
-    const latest_sub_obj = await getSubscription(sub.id);
+exports.getVideosForSub = async (sub, user_uid = null) => {
+    const latest_sub_obj = await exports.getSubscription(sub.id);
     if (!latest_sub_obj || latest_sub_obj['downloading']) {
         return false;
     }
@@ -305,7 +307,7 @@ async function handleOutputJSON(output, sub, user_uid) {
     }
 
     const files_to_download = await getFilesToDownload(sub, output_jsons);
-    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+    const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
 
     for (let j = 0; j < files_to_download.length; j++) {
         const file_to_download = files_to_download[j];
@@ -316,7 +318,7 @@ async function handleOutputJSON(output, sub, user_uid) {
     return files_to_download;
 }
 
-function generateOptionsForSubscriptionDownload(sub, user_uid) {
+exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -371,10 +373,13 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     // skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
     const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
-    logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
-    const archive_path = path.join(appendedBasePath, 'archive.txt');
-    await fs.writeFile(archive_path, archive_text);
-    downloadConfig.push('--download-archive', archive_path);
+    const archive_count = archive_text.split('\n').length - 1;
+    if (archive_count > 0) {
+        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
+        const archive_path = path.join(appendedBasePath, 'archive.txt');
+        await fs.writeFile(archive_path, archive_text);
+        downloadConfig.push('--download-archive', archive_path);
+    }
 
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
@@ -440,35 +445,36 @@ async function getFilesToDownload(sub, output_jsons) {
 }
 
 
-async function getSubscriptions(user_uid = null) {
+exports.getSubscriptions = async (user_uid = null) => {
     return await db_api.getRecords('subscriptions', {user_uid: user_uid});
 }
 
-async function getAllSubscriptions() {
+exports.getAllSubscriptions = async () => {
     const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
 }
 
-async function getSubscription(subID) {
+exports.getSubscription = async (subID) => {
     // stringify and parse because we may override the 'downloading' property
     const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
     // now with the download_queue, we may need to override 'downloading'
-    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
     if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
     return sub;
 }
 
-async function getSubscriptionByName(subName, user_uid = null) {
+exports.getSubscriptionByName = async (subName, user_uid = null) => {
     return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
 }
 
-async function updateSubscription(sub) {
+exports.updateSubscription = async (sub) => {
     await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
+    exports.writeSubscriptionMetadata(sub);
     return true;
 }
 
-async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
+exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
     subs.forEach(async sub => {
         await updateSubscriptionProperty(sub, assignment_obj);
     });
@@ -480,6 +486,16 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
+exports.writeSubscriptionMetadata = (sub) => {
+    let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
+                                : config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const appendedBasePath = getAppendedBasePath(sub, basePath);
+    const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
+    
+    fs.ensureDirSync(appendedBasePath);
+    fs.writeJSONSync(metadata_path, sub);
+}
+
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     if (!sub_files) return;
@@ -534,17 +550,3 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
 function getAppendedBasePath(sub, base_path) {
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
-
-module.exports = {
-    getSubscription        : getSubscription,
-    getSubscriptionByName  : getSubscriptionByName,
-    getSubscriptions       : getSubscriptions,
-    getAllSubscriptions    : getAllSubscriptions,
-    updateSubscription     : updateSubscription,
-    subscribe              : subscribe,
-    unsubscribe            : unsubscribe,
-    deleteSubscriptionFile : deleteSubscriptionFile,
-    getVideosForSub        : getVideosForSub,
-    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
-    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
-}

backend/tasks.js

@@ -3,10 +3,17 @@ const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
 const archive_api = require('./archive');
 const files_api = require('./files');
+const subscriptions_api = require('./subscriptions');
+const config_api = require('./config');
+const auth_api = require('./authentication/auth');
+const utils = require('./utils');
+const logger = require('./logger');
+const CONSTS = require('./consts');
 
 const fs = require('fs-extra');
-const logger = require('./logger');
+const path = require('path');
 const scheduler = require('node-schedule');
+const { uuid } = require('uuidv4');
 
 const TASKS = {
     backup_local_db: {
@@ -47,6 +54,11 @@ const TASKS = {
         run: archive_api.importArchives,
         title: 'Import legacy archives',
         job: null
+    },
+    rebuild_database: {
+        run: rebuildDB,
+        title: 'Rebuild database',
+        job: null
     }
 }
 
@@ -265,4 +277,68 @@ async function autoDeleteFiles(data) {
     }
 }
 
+async function rebuildDB() {
+    await db_api.backupDB();
+    let subs_to_add = await guessSubscriptions(false);
+    subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
+    const users_to_add = await guessUsers();
+    for (const user_to_add of users_to_add) {
+        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        
+        const user_exists = await db_api.getRecord('users', {uid: user_to_add});
+        if (!user_exists) {
+            await auth_api.registerUser(user_to_add, user_to_add, 'password');
+            logger.info(`Regenerated user ${user_to_add}`);
+        }
+        
+        const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
+        const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
+        subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
+    }
+
+    for (const sub_to_add of subs_to_add) {
+        const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
+        // TODO: we shouldn't be creating this here
+        const new_sub = Object.assign({}, sub_to_add, {paused: true});
+        if (!sub_exists) {
+            await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
+            logger.info(`Regenerated subscription ${sub_to_add['name']}`);
+        }
+    }
+
+    logger.info(`Importing unregistered files`);
+    await files_api.importUnregisteredFiles();
+}
+
+const guessUsers = async () => {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
+    return userPaths.map(userPath => path.basename(userPath));
+}
+
+const guessSubscriptions = async (isPlaylist, basePath = null) => {
+    const guessed_subs = [];
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
+    const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
+
+    const subs = await utils.getDirectoriesInDirectory(subsPath);
+    for (const subPath of subs) {
+        const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
+        if (!fs.existsSync(sub_backup_path)) continue;
+
+        try {
+            const sub_backup = fs.readJSONSync(sub_backup_path)
+            delete sub_backup['_id'];
+            guessed_subs.push(sub_backup);
+        } catch(err) {
+            logger.warn(`Failed to reimport subscription in path ${subPath}`)
+            logger.warn(err);
+        }
+    }
+
+    return guessed_subs;
+}
+
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -337,16 +337,22 @@ describe('Database', async function() {
 });
 
 describe('Multi User', async function() {
-    let user = null;
-    const user_to_test = 'admin';
-    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
-    const playlist_to_test = 'ysabVZz4x';
+    const user_to_test = 'test_user';
+    const user_password = 'test_pass';
+    const sub_to_test = '';
+    const playlist_to_test = '';
     beforeEach(async function() {
         await db_api.connectToDB();
-        user = await auth_api.login('admin', 'pass');
+        await auth_api.deleteUser(user_to_test);
     });
-    describe('Authentication', function() {
-        it('login', async function() {
+    describe('Basic', function() {
+        it('Register', async function() {
+            const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
+            assert(user);
+        });
+        it('Login', async function() {
+            await auth_api.registerUser(user_to_test, user_to_test, user_password);
+            const user = await auth_api.login(user_to_test, user_password);
             assert(user);
         });
     });
@@ -362,14 +368,14 @@ describe('Multi User', async function() {
         });
 
         it('Video access - disallowed', async function() {
-            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
-            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(!video_obj);
         });
 
         it('Video access - allowed', async function() {
             await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
-            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(video_obj);
         });
     });

backend/utils.js

@@ -519,6 +519,17 @@ exports.convertFlatObjectToNestedObject = (obj) => {
     return result;
 }
 
+exports.getDirectoriesInDirectory = async (basePath) => {
+    try {
+        const files = await fs.readdir(basePath, { withFileTypes: true });
+        return files
+            .filter((file) => file.isDirectory())
+            .map((file) => path.join(basePath, file.name));
+    } catch (err) {
+        return [];
+    }
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.3.1"
+appVersion: "4.3.2"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.3.1",
+  "version": "4.3.2",
   "license": "MIT",
   "scripts": {
     "ng": "ng",

src/api-types/index.ts

@@ -104,6 +104,7 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
 export type { SuccessObject } from './models/SuccessObject';
 export type { TableInfo } from './models/TableInfo';
 export type { Task } from './models/Task';
+export { TaskType } from './models/TaskType';
 export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
 export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
 export type { TransferDBRequest } from './models/TransferDBRequest';

src/api-types/models/Config.ts

@@ -3,5 +3,5 @@
 /* eslint-disable */
 
 export type Config = {
-    YoutubeDLMaterial: Record<string, any>;
+    YoutubeDLMaterial: any;
 };

src/api-types/models/Download.ts

@@ -26,5 +26,5 @@ export type Download = {
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
-    prefetched_info?: Record<string, any>;
+    prefetched_info?: any;
 };

src/api-types/models/GetFileFormatsResponse.ts

@@ -5,6 +5,6 @@
 export type GetFileFormatsResponse = {
     success: boolean;
     result: {
-formats?: Array<Record<string, any>>;
+formats?: Array<any>;
 };
 };

src/api-types/models/GetSubscriptionResponse.ts

@@ -6,5 +6,5 @@ import type { Subscription } from './Subscription';
 
 export type GetSubscriptionResponse = {
     subscription: Subscription;
-    files: Array<Record<string, any>>;
+    files: Array<any>;
 };

src/api-types/models/GetTaskRequest.ts

@@ -2,6 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { TaskType } from './TaskType';
+
 export type GetTaskRequest = {
-    task_key: string;
+    task_key: TaskType;
 };

src/api-types/models/Notification.ts

@@ -11,6 +11,6 @@ export type Notification = {
     user_uid?: string;
     action?: Array<NotificationAction>;
     read: boolean;
-    data?: Record<string, any>;
+    data?: any;
     timestamp: number;
 };

src/api-types/models/Subscription.ts

@@ -15,5 +15,5 @@ export type Subscription = {
     timerange?: string;
     custom_args?: string;
     custom_output?: string;
-    videos: Array<Record<string, any>>;
+    videos: Array<any>;
 };

src/api-types/models/Task.ts

@@ -3,16 +3,17 @@
 /* eslint-disable */
 
 import type { Schedule } from './Schedule';
+import type { TaskType } from './TaskType';
 
 export type Task = {
-    key: string;
+    key: TaskType;
     title?: string;
     last_ran: number;
     last_confirmed: number;
     running: boolean;
     confirming: boolean;
-    data: Record<string, any>;
+    data: any;
     error: string;
     schedule: Schedule;
-    options?: Record<string, any>;
+    options?: any;
 };

src/api-types/models/TaskType.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum TaskType {
+    BACKUP_LOCAL_DB = 'backup_local_db',
+    MISSING_FILES_CHECK = 'missing_files_check',
+    MISSING_DB_RECORDS = 'missing_db_records',
+    DUPLICATE_FILES_CHECK = 'duplicate_files_check',
+    YOUTUBEDL_UPDATE_CHECK = 'youtubedl_update_check',
+    DELETE_OLD_FILES = 'delete_old_files',
+    IMPORT_LEGACY_ARCHIVES = 'import_legacy_archives',
+    REBUILD_DATABASE = 'rebuild_database',
+}

src/api-types/models/UpdateFileRequest.ts

@@ -10,5 +10,5 @@ export type UpdateFileRequest = {
     /**
      * Object with fields to update as keys and their new values
      */
-    change_obj: Record<string, any>;
+    change_obj: any;
 };

src/api-types/models/UpdateTaskDataRequest.ts

@@ -2,7 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { TaskType } from './TaskType';
+
 export type UpdateTaskDataRequest = {
-    task_key: string;
-    new_data: Record<string, any>;
+    task_key: TaskType;
+    new_data: any;
 };

src/api-types/models/UpdateTaskOptionsRequest.ts

@@ -2,7 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { TaskType } from './TaskType';
+
 export type UpdateTaskOptionsRequest = {
-    task_key: string;
-    new_options: Record<string, any>;
+    task_key: TaskType;
+    new_options: any;
 };

src/api-types/models/UpdateTaskScheduleRequest.ts

@@ -3,8 +3,9 @@
 /* eslint-disable */
 
 import type { Schedule } from './Schedule';
+import type { TaskType } from './TaskType';
 
 export type UpdateTaskScheduleRequest = {
-    task_key: string;
+    task_key: TaskType;
     new_schedule: Schedule;
 };

src/app/components/task-settings/task-settings.component.ts

@@ -1,6 +1,6 @@
 import { Component, Inject } from '@angular/core';
 import { MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { Task } from 'api-types';
+import { Task, TaskType } from 'api-types';
 import { PostsService } from 'app/posts.services';
 
 @Component({
@@ -9,7 +9,7 @@ import { PostsService } from 'app/posts.services';
   styleUrls: ['./task-settings.component.scss']
 })
 export class TaskSettingsComponent {
-  task_key: string;
+  task_key: TaskType;
   new_options = {};
   task: Task = null;
 

src/app/components/tasks/tasks.component.html

@@ -49,7 +49,7 @@
             <mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
             <mat-cell *matCellDef="let element">
                 <div class="container">
-                    <div class="row justify-content-center">
+                    <div class="row justify-content-start">
                         <div *ngIf="element.data?.uids?.length > 0 || (!element.data?.uids && element.data)" class="col-12 mt-2" style="display: flex; justify-content: center;">
                             <ng-container>
                                 <button (click)="confirmTask(element.key)" [disabled]="element.running || element.confirming" mat-stroked-button>

src/app/components/tasks/tasks.component.ts

@@ -7,7 +7,7 @@ import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialo
 import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component';
 import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
 import { PostsService } from 'app/posts.services';
-import { Task } from 'api-types';
+import { Task, TaskType } from 'api-types';
 import { TaskSettingsComponent } from '../task-settings/task-settings.component';
 import { Clipboard } from '@angular/cdk/clipboard';
 
@@ -28,6 +28,15 @@ export class TasksComponent implements OnInit {
 
   db_backups = [];
 
+  TASKS_TO_REQUIRE_DIALOG: { [key in TaskType]? : {dialogTitle: string, dialogText: string, submitText: string, warnSubmitColor: boolean}} = {
+    [TaskType.REBUILD_DATABASE]: {
+      dialogTitle: $localize`Rebuild database`,
+      dialogText: $localize`Are you sure you want to rebuild the database? All missing users, subscriptions, and files will be reimported. Note that if missing users are detected, they will be created with the password: 'password'. A backup of your current database will be created.`,
+      submitText: $localize`Rebuild database`,
+      warnSubmitColor: false
+    }
+  }
+
   @ViewChild(MatPaginator) paginator: MatPaginator;
   @ViewChild(MatSort) sort: MatSort;
 
@@ -79,7 +88,29 @@ export class TasksComponent implements OnInit {
     });
   }
 
-  runTask(task_key: string): void {
+  runTask(task_key: TaskType): void {
+    const taskToRequireDialog = this.TASKS_TO_REQUIRE_DIALOG[task_key];
+    if (taskToRequireDialog) {
+      const dialogRef = this.dialog.open(ConfirmDialogComponent, {
+        data: {
+          dialogTitle: taskToRequireDialog['dialogTitle'],
+          dialogText: taskToRequireDialog['dialogText'],
+          submitText: taskToRequireDialog['submitText'],
+          warnSubmitColor: taskToRequireDialog['warnSubmitColor']
+        }
+      });
+      dialogRef.afterClosed().subscribe(confirmed => {
+        if (confirmed) {
+          this._runTask(task_key);
+        }
+      });
+      return;
+    }
+
+    this._runTask(task_key);
+  }
+
+  _runTask(task_key: TaskType): void {
     this.postsService.runTask(task_key).subscribe(res => {
       this.getTasks();
       this.getDBBackups();
@@ -91,7 +122,7 @@ export class TasksComponent implements OnInit {
     });
   }
 
-  confirmTask(task_key: string): void {
+  confirmTask(task_key: TaskType): void {
     this.postsService.confirmTask(task_key).subscribe(res => {
       this.getTasks();
       if (res['success']) this.postsService.openSnackBar($localize`Successfully confirmed task!`);

src/app/consts.ts

@@ -1 +1 @@
-export const CURRENT_VERSION = 'v4.3.1';
+export const CURRENT_VERSION = 'v4.3.2';

src/app/posts.services.ts

@@ -112,7 +112,8 @@ import {
     ImportArchiveRequest,
     Archive,
     Subscription,
-    RestartDownloadResponse
+    RestartDownloadResponse,
+    TaskType
 } from '../api-types';
 import { isoLangs } from './dialogs/user-profile-dialog/locales_list';
 import { Title } from '@angular/platform-browser';
@@ -640,32 +641,32 @@ export class PostsService implements CanActivate {
         return this.http.post<SuccessObject>(this.path + 'resetTasks', {}, this.httpOptions);
     }
 
-    getTask(task_key: string) {
+    getTask(task_key: TaskType) {
         const body: GetTaskRequest = {task_key: task_key};
         return this.http.post<GetTaskResponse>(this.path + 'getTask', body, this.httpOptions);
     }
 
-    runTask(task_key: string) {
+    runTask(task_key: TaskType) {
         const body: GetTaskRequest = {task_key: task_key};
         return this.http.post<SuccessObject>(this.path + 'runTask', body, this.httpOptions);
     }
 
-    confirmTask(task_key: string) {
+    confirmTask(task_key: TaskType) {
         const body: GetTaskRequest = {task_key: task_key};
         return this.http.post<SuccessObject>(this.path + 'confirmTask', body, this.httpOptions);
     }
 
-    updateTaskSchedule(task_key: string, schedule: Schedule) {
+    updateTaskSchedule(task_key: TaskType, schedule: Schedule) {
         const body: UpdateTaskScheduleRequest = {task_key: task_key, new_schedule: schedule};
         return this.http.post<SuccessObject>(this.path + 'updateTaskSchedule', body, this.httpOptions);
     }
 
-    updateTaskData(task_key: string, data: any) {
+    updateTaskData(task_key: TaskType, data: any) {
         const body: UpdateTaskDataRequest = {task_key: task_key, new_data: data};
         return this.http.post<SuccessObject>(this.path + 'updateTaskData', body, this.httpOptions);
     }
 
-    updateTaskOptions(task_key: string, options: any) {
+    updateTaskOptions(task_key: TaskType, options: any) {
         const body: UpdateTaskOptionsRequest = {task_key: task_key, new_options: options};
         return this.http.post<SuccessObject>(this.path + 'updateTaskOptions', body, this.httpOptions);
     }

src/assets/i18n/messages.zh.xlf

@@ -1432,7 +1432,7 @@
       </trans-unit>
       <trans-unit id="994363f08f9fbfa3b3994ff7b35c6904fdff18d8" datatype="html">
         <source>Profile</source>
-        <target state="translated">资料</target>
+        <target state="translated">个人资料</target>
         <context-group purpose="location">
           <context context-type="sourcefile">app/app.component.html</context>
           <context context-type="linenumber">19</context>
@@ -5015,6 +5015,24 @@
         </context-group>
         <note priority="1" from="description">Use gotify API setting</note>
       </trans-unit>
+      <trans-unit id="7cedb649779673568447b994463b2882c4e0436a" datatype="html">
+        <source>Slack Webhook URL</source>
+        <target state="translated">Slack Webhook 网址</target>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/settings/settings.component.html</context>
+          <context context-type="linenumber">397</context>
+        </context-group>
+        <note priority="1" from="description">Slack Webhook URL</note>
+      </trans-unit>
+      <trans-unit id="3264d82792954815be755b3da01e2625458711dc" datatype="html">
+        <source>Discord Webhook URL</source>
+        <target state="translated">Discord Webhook 网址</target>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/settings/settings.component.html</context>
+          <context context-type="linenumber">390</context>
+        </context-group>
+        <note priority="1" from="description">Discord Webhook URL</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>