From bad60807307a5ca6730dbb975c9b41e6f994f878 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Mon, 8 May 2023 02:00:43 -0400 Subject: [PATCH] Adds task to rebuild database --- backend/subscriptions.js | 6 +-- backend/tasks.js | 82 +++++++++++++++++++++++++++++++++++++++- backend/utils.js | 11 ++++++ 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/backend/subscriptions.js b/backend/subscriptions.js index a6d2b18..8b4b905 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -19,7 +19,7 @@ async function subscribe(sub, user_uid = null) { }; return new Promise(async resolve => { // sub should just have url and name. here we will get isPlaylist and path - sub.isPlaylist = sub.url.includes('playlist'); + sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist'); sub.videos = []; let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid})); @@ -37,7 +37,7 @@ async function subscribe(sub, user_uid = null) { let success = await getSubscriptionInfo(sub); if (success) { - getVideosForSub(sub, user_uid); + if (!sub.paused) getVideosForSub(sub, user_uid); } else { logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') } @@ -454,7 +454,7 @@ async function getSubscription(subID) { // stringify and parse because we may override the 'downloading' property const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID}))); // now with the download_queue, we may need to override 'downloading' - const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true); + const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true); if (!sub['downloading']) sub['downloading'] = current_downloads > 0; return sub; } diff --git a/backend/tasks.js b/backend/tasks.js index e7a3493..a86a9d6 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -3,10 +3,16 @@ const notifications_api = require('./notifications'); const youtubedl_api = require('./youtube-dl'); const archive_api = require('./archive'); const files_api = require('./files'); +const subscriptions_api = require('./subscriptions'); +const config_api = require('./config'); +const auth_api = require('./authentication/auth'); +const utils = require('./utils'); +const logger = require('./logger'); const fs = require('fs-extra'); -const logger = require('./logger'); +const path = require('path'); const scheduler = require('node-schedule'); +const { uuid } = require('uuidv4'); const TASKS = { backup_local_db: { @@ -47,6 +53,11 @@ const TASKS = { run: archive_api.importArchives, title: 'Import legacy archives', job: null + }, + rebuild_database: { + run: rebuildDB, + title: 'Rebuild database', + job: null } } @@ -265,4 +276,73 @@ async function autoDeleteFiles(data) { } } +async function rebuildDB() { + await db_api.backupDB(); + let subs_to_add = await guessSubscriptions(false); + subs_to_add = subs_to_add.concat(await guessSubscriptions(true)); + const users_to_add = await guessUsers(); + for (const user_to_add of users_to_add) { + const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + + const user_exists = await db_api.getRecord('users', {uid: user_to_add}); + if (!user_exists) await auth_api.registerUser(user_to_add, user_to_add, ''); + else logger.info(`Regenerated user ${user_to_add}`); + + const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add); + const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add); + subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs); + } + + for (const sub_to_add of subs_to_add) { + const sub_exists = await db_api.getRecord('subscriptions', {name: sub_to_add['name']}); + // TODO: we shouldn't be creating this here + const new_sub = { + name: sub_to_add['name'], + url: sub_to_add['url'], + maxQuality: 'best', + id: uuid(), + user_uid: sub_to_add['user_uid'], + type: sub_to_add['type'], + paused: true + }; + if (!sub_exists) await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid']); + else logger.info(`Regenerated subscription ${sub_to_add['name']}`); + } +} + +const guessUsers = async () => { + const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder); + return userPaths.map(userPath => path.basename(userPath)); +} + +const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null) => { + const guessed_subs = []; + const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder; + const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels'); + + const subs = await utils.getDirectoriesInDirectory(subsPath); + for (const subPath of subs) { + const audio_files = await utils.getDownloadedFilesByType(subPath, 'audio', true); + const video_files = await utils.getDownloadedFilesByType(subPath, 'video', true); + const files = audio_files.concat(video_files); + + if (files.length === 0) continue; + + const sample_file = files[0]; + const url = sample_file['channel_url']; + guessed_subs.push({ + url: url, + name: path.basename(subPath), + user_uid: user_uid, + type: video_files.length !== 0 ? 'video' : 'audio', + isPlaylist: isPlaylist + }); + } + + return guessed_subs; +} + exports.TASKS = TASKS; \ No newline at end of file diff --git a/backend/utils.js b/backend/utils.js index eafb946..157b6f0 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -519,6 +519,17 @@ exports.convertFlatObjectToNestedObject = (obj) => { return result; } +exports.getDirectoriesInDirectory = async (basePath) => { + try { + const files = await fs.readdir(basePath, { withFileTypes: true }); + return files + .filter((file) => file.isDirectory()) + .map((file) => path.join(basePath, file.name)); + } catch (err) { + return []; + } +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {