TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rebuild database task (#900)

Browse Source 
* Improved tests for multi-user mode
* Adds task to rebuild database
* Updated subscriptions.js export syntax
* Subscription metadata is now backed up
* Added typing to task key
* Updated api models
* Tasks actions styling update
This commit is contained in:
Tzahi12345
2023-05-23 22:37:25 -04:00
committed by GitHub
parent 441131e930
commit c207e56855
26 changed files with 304 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
backend/app.js
View File

@@ -535,6 +535,7 @@ async function loadConfig() {
if (allowSubscriptions) {
// set downloading to false
let subscriptions = await subscriptions_api.getAllSubscriptions();
subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
// runs initially, then runs every ${subscriptionCheckInterval} seconds
const watchSubscriptionsInterval = function() {
@@ -1928,9 +1929,34 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
// user authentication
app.post('/api/auth/register'
, optionalJwt
, auth_api.registerUser);
app.post('/api/auth/register', optionalJwt, async (req, res) => {
const userid = req.body.userid;
const username = req.body.username;
const plaintextPassword = req.body.password;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
res.sendStatus(409);
return;
}
if (plaintextPassword === "") {
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
res.sendStatus(409);
return;
}
const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
if (!new_user) {
res.sendStatus(409);
return;
}
res.send({
user: new_user
});
});
app.post('/api/auth/login'
, auth_api.passport.authenticate(['local', 'ldapauth'], {})
, auth_api.generateJWT
@@ -1982,18 +2008,7 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
app.post('/api/deleteUser', optionalJwt, async (req, res) => {
let uid = req.body.uid;
try {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
}
const success = await auth_api.deleteUser(uid);
res.send({success: success});
} catch (err) {
logger.error(err);

94
backend/authentication/auth.js
View File

@@ -1,11 +1,13 @@
const config_api = require('../config');
const consts = require('../consts');
const CONSTS = require('../consts');
const logger = require('../logger');
const db_api = require('../db');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
const fs = require('fs-extra');
const path = require('path');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -16,7 +18,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
let saltRounds = 10;
exports.initialize = function () {
/*************************
@@ -31,8 +33,6 @@ exports.initialize = function () {
});
}
saltRounds = 10;
// Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
if (!(+JWT_EXPIRATION)) {
@@ -68,7 +68,7 @@ exports.initialize = function () {
const setupRoles = async () => {
const required_roles = {
admin: {
permissions: consts.AVAILABLE_PERMISSIONS
permissions: CONSTS.AVAILABLE_PERMISSIONS
},
user: {
permissions: [
@@ -106,55 +106,41 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
res.sendStatus(409);
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
return;
exports.registerUser = async (userid, username, plaintextPassword) => {
const hash = await bcrypt.hash(plaintextPassword, saltRounds);
const new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
return null;
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
return null;
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
return new_user;
}
}
if (plaintextPassword === "") {
res.sendStatus(400);
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
return;
exports.deleteUser = async (uid) => {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
}
bcrypt.hash(plaintextPassword, saltRounds)
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
});
}
})
.then(function(result) {
})
.catch(function(err) {
logger.error(err);
if( err.code == 'ER_DUP_ENTRY' ) {
res.status(409).send('UserId already taken');
} else {
res.sendStatus(409);
}
});
return success;
}
/***************************************
@@ -235,7 +221,7 @@ exports.returnAuthResponse = async function(req, res) {
user: req.user,
token: req.token,
permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS']
available_permissions: CONSTS.AVAILABLE_PERMISSIONS
});
}
@@ -319,7 +305,7 @@ exports.getUserVideos = async function(user_uid, type) {
}
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {file_uid: file_uid});
let file = await db_api.getRecord('files', {uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -406,8 +392,8 @@ exports.userPermissions = async function(user_uid) {
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];
for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

2
backend/consts.js
View File

@@ -347,6 +347,8 @@ const YTDL_ARGS_WITH_VALUES = [
'--convert-subs'
];
exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
// we're using a Set here for performance
exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);

59
backend/subscriptions.js
View File

@@ -6,20 +6,21 @@ const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const debugMode = process.env.YTDL_MODE === 'debug';
const db_api = require('./db');
const downloader_api = require('./downloader');
async function subscribe(sub, user_uid = null) {
exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
const result_obj = {
success: false,
error: ''
};
return new Promise(async resolve => {
// sub should just have url and name. here we will get isPlaylist and path
sub.isPlaylist = sub.url.includes('playlist');
sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
sub.videos = [];
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -34,10 +35,11 @@ async function subscribe(sub, user_uid = null) {
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub);
let success = skip_get_info ? true : await getSubscriptionInfo(sub);
exports.writeSubscriptionMetadata(sub);
if (success) {
getVideosForSub(sub, user_uid);
if (!sub.paused) exports.getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
}
@@ -109,7 +111,7 @@ async function getSubscriptionInfo(sub) {
});
}
async function unsubscribe(sub, deleteMode, user_uid = null) {
exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -148,7 +150,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
await db_api.removeAllRecords('archives', {sub_id: sub.id});
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
if (typeof sub === 'string') {
// TODO: fix bad workaround where sub is a sub_id
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
@@ -216,8 +218,8 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
}
}
async function getVideosForSub(sub, user_uid = null) {
const latest_sub_obj = await getSubscription(sub.id);
exports.getVideosForSub = async (sub, user_uid = null) => {
const latest_sub_obj = await exports.getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -305,7 +307,7 @@ async function handleOutputJSON(output, sub, user_uid) {
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
@@ -316,7 +318,7 @@ async function handleOutputJSON(output, sub, user_uid) {
return files_to_download;
}
function generateOptionsForSubscriptionDownload(sub, user_uid) {
exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -443,35 +445,36 @@ async function getFilesToDownload(sub, output_jsons) {
}
async function getSubscriptions(user_uid = null) {
exports.getSubscriptions = async (user_uid = null) => {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
async function getAllSubscriptions() {
exports.getAllSubscriptions = async () => {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}
async function getSubscription(subID) {
exports.getSubscription = async (subID) => {
// stringify and parse because we may override the 'downloading' property
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
// now with the download_queue, we may need to override 'downloading'
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
return sub;
}
async function getSubscriptionByName(subName, user_uid = null) {
exports.getSubscriptionByName = async (subName, user_uid = null) => {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
async function updateSubscription(sub) {
exports.updateSubscription = async (sub) => {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
exports.writeSubscriptionMetadata(sub);
return true;
}
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj);
});
@@ -483,6 +486,14 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
return true;
}
exports.writeSubscriptionMetadata = (sub) => {
let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
fs.writeJSONSync(metadata_path, sub);
}
async function setFreshUploads(sub) {
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
if (!sub_files) return;
@@ -537,17 +548,3 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
getSubscriptions : getSubscriptions,
getAllSubscriptions : getAllSubscriptions,
updateSubscription : updateSubscription,
subscribe : subscribe,
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
}

78
backend/tasks.js
View File

@@ -3,10 +3,17 @@ const notifications_api = require('./notifications');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
const subscriptions_api = require('./subscriptions');
const config_api = require('./config');
const auth_api = require('./authentication/auth');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const fs = require('fs-extra');
const logger = require('./logger');
const path = require('path');
const scheduler = require('node-schedule');
const { uuid } = require('uuidv4');
const TASKS = {
backup_local_db: {
@@ -47,6 +54,11 @@ const TASKS = {
run: archive_api.importArchives,
title: 'Import legacy archives',
job: null
},
rebuild_database: {
run: rebuildDB,
title: 'Rebuild database',
job: null
}
}
@@ -265,4 +277,68 @@ async function autoDeleteFiles(data) {
}
}
async function rebuildDB() {
await db_api.backupDB();
let subs_to_add = await guessSubscriptions(false);
subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
const users_to_add = await guessUsers();
for (const user_to_add of users_to_add) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_exists = await db_api.getRecord('users', {uid: user_to_add});
if (!user_exists) {
await auth_api.registerUser(user_to_add, user_to_add, 'password');
logger.info(`Regenerated user ${user_to_add}`);
}
const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
}
for (const sub_to_add of subs_to_add) {
const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
// TODO: we shouldn't be creating this here
const new_sub = Object.assign({}, sub_to_add, {paused: true});
if (!sub_exists) {
await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
logger.info(`Regenerated subscription ${sub_to_add['name']}`);
}
}
logger.info(`Importing unregistered files`);
await files_api.importUnregisteredFiles();
}
const guessUsers = async () => {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
return userPaths.map(userPath => path.basename(userPath));
}
const guessSubscriptions = async (isPlaylist, basePath = null) => {
const guessed_subs = [];
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
const subs = await utils.getDirectoriesInDirectory(subsPath);
for (const subPath of subs) {
const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
if (!fs.existsSync(sub_backup_path)) continue;
try {
const sub_backup = fs.readJSONSync(sub_backup_path)
delete sub_backup['_id'];
guessed_subs.push(sub_backup);
} catch(err) {
logger.warn(`Failed to reimport subscription in path ${subPath}`)
logger.warn(err);
}
}
return guessed_subs;
}
exports.TASKS = TASKS;

26
backend/test/tests.js
View File

@@ -337,16 +337,22 @@ describe('Database', async function() {
});
describe('Multi User', async function() {
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
const user_to_test = 'test_user';
const user_password = 'test_pass';
const sub_to_test = '';
const playlist_to_test = '';
beforeEach(async function() {
await db_api.connectToDB();
user = await auth_api.login('admin', 'pass');
await auth_api.deleteUser(user_to_test);
});
describe('Authentication', function() {
it('login', async function() {
describe('Basic', function() {
it('Register', async function() {
const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
assert(user);
});
it('Login', async function() {
await auth_api.registerUser(user_to_test, user_to_test, user_password);
const user = await auth_api.login(user_to_test, user_password);
assert(user);
});
});
@@ -362,14 +368,14 @@ describe('Multi User', async function() {
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
assert(video_obj);
});
});

11
backend/utils.js
View File

@@ -519,6 +519,17 @@ exports.convertFlatObjectToNestedObject = (obj) => {
return result;
}
exports.getDirectoriesInDirectory = async (basePath) => {
try {
const files = await fs.readdir(basePath, { withFileTypes: true });
return files
.filter((file) => file.isDirectory())
.map((file) => path.join(basePath, file.name));
} catch (err) {
return [];
}
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {