diff --git a/backend/app.js b/backend/app.js index 11ac95b..049d9bc 100644 --- a/backend/app.js +++ b/backend/app.js @@ -535,6 +535,7 @@ async function loadConfig() { if (allowSubscriptions) { // set downloading to false let subscriptions = await subscriptions_api.getAllSubscriptions(); + subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub)); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds const watchSubscriptionsInterval = function() { diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 69fa6c7..0ab6e00 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -1,5 +1,5 @@ const config_api = require('../config'); -const consts = require('../consts'); +const CONSTS = require('../consts'); const logger = require('../logger'); const db_api = require('../db'); @@ -68,14 +68,7 @@ exports.initialize = function () { const setupRoles = async () => { const required_roles = { admin: { - permissions: [ - 'filemanager', - 'settings', - 'subscriptions', - 'sharing', - 'advanced_download', - 'downloads_manager' - ] + permissions: CONSTS.AVAILABLE_PERMISSIONS }, user: { permissions: [ diff --git a/backend/consts.js b/backend/consts.js index 4461819..6a36d10 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -347,6 +347,8 @@ const YTDL_ARGS_WITH_VALUES = [ '--convert-subs' ]; +exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json' + // we're using a Set here for performance exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index b48f234..8522f78 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -6,13 +6,14 @@ const config_api = require('./config'); const archive_api = require('./archive'); const utils = require('./utils'); const logger = require('./logger'); +const CONSTS = require('./consts'); const debugMode = process.env.YTDL_MODE === 'debug'; const db_api = require('./db'); const downloader_api = require('./downloader'); -exports.subscribe = async (sub, user_uid = null) => { +exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => { const result_obj = { success: false, error: '' @@ -34,7 +35,8 @@ exports.subscribe = async (sub, user_uid = null) => { sub['user_uid'] = user_uid ? user_uid : undefined; await db_api.insertRecordIntoTable('subscriptions', sub); - let success = await getSubscriptionInfo(sub); + let success = skip_get_info ? true : await getSubscriptionInfo(sub); + exports.writeSubscriptionMetadata(sub); if (success) { if (!sub.paused) exports.getVideosForSub(sub, user_uid); @@ -465,6 +467,7 @@ exports.getSubscriptionByName = async (subName, user_uid = null) => { exports.updateSubscription = async (sub) => { await db_api.updateRecord('subscriptions', {id: sub.id}, sub); + exports.writeSubscriptionMetadata(sub); return true; } @@ -480,6 +483,14 @@ async function updateSubscriptionProperty(sub, assignment_obj) { return true; } +exports.writeSubscriptionMetadata = (sub) => { + let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions') + : config_api.getConfigItem('ytdl_subscriptions_base_path'); + const appendedBasePath = getAppendedBasePath(sub, basePath); + const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH); + fs.writeJSONSync(metadata_path, sub); +} + async function setFreshUploads(sub) { const sub_files = await db_api.getRecords('files', {sub_id: sub.id}); if (!sub_files) return; diff --git a/backend/tasks.js b/backend/tasks.js index 6f487cc..11c2b70 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -8,6 +8,7 @@ const config_api = require('./config'); const auth_api = require('./authentication/auth'); const utils = require('./utils'); const logger = require('./logger'); +const CONSTS = require('./consts'); const fs = require('fs-extra'); const path = require('path'); @@ -279,6 +280,7 @@ async function autoDeleteFiles(data) { async function rebuildDB() { await db_api.backupDB(); let subs_to_add = await guessSubscriptions(false); + let subs_to_add = await guessSubscriptions(true); subs_to_add = subs_to_add.concat(await guessSubscriptions(true)); const users_to_add = await guessUsers(); for (const user_to_add of users_to_add) { @@ -298,17 +300,9 @@ async function rebuildDB() { for (const sub_to_add of subs_to_add) { const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid'])); // TODO: we shouldn't be creating this here - const new_sub = { - name: sub_to_add['name'], - url: sub_to_add['url'], - maxQuality: 'best', - id: uuid(), - user_uid: sub_to_add['user_uid'], - type: sub_to_add['type'], - paused: true - }; + const new_sub = Object.assign({}, sub_to_add, {paused: true}); if (!sub_exists) { - await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid']); + await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true); logger.info(`Regenerated subscription ${sub_to_add['name']}`); } } @@ -323,7 +317,7 @@ const guessUsers = async () => { return userPaths.map(userPath => path.basename(userPath)); } -const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null) => { +const guessSubscriptions = async (isPlaylist, basePath = null) => { const guessed_subs = []; const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path'); @@ -332,21 +326,17 @@ const guessSubscriptions = async (isPlaylist, basePath = null, user_uid = null) const subs = await utils.getDirectoriesInDirectory(subsPath); for (const subPath of subs) { - const audio_files = await utils.getDownloadedFilesByType(subPath, 'audio', true); - const video_files = await utils.getDownloadedFilesByType(subPath, 'video', true); - const files = audio_files.concat(video_files); - - if (files.length === 0) continue; + const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH); + if (!fs.existsSync(sub_backup_path)) continue; - const sample_file = files[0]; - const url = sample_file['channel_url']; - guessed_subs.push({ - url: url, - name: path.basename(subPath), - user_uid: user_uid, - type: video_files.length !== 0 ? 'video' : 'audio', - isPlaylist: isPlaylist - }); + try { + const sub_backup = fs.readJSONSync(sub_backup_path) + delete sub_backup['_id']; + guessed_subs.push(sub_backup); + } catch(err) { + logger.warn(`Failed to reimport subscription in path ${subPath}`) + logger.warn(err); + } } return guessed_subs;