mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-28 15:40:56 +03:00
Changed DB structure again
Added support for MongoDB Added tests relating to new DB system Category rules are now case insensitive Fixed playlist modification change state
This commit is contained in:
Isaac Abadi
766
backend/db.js
766
backend/db.js
@@ -3,22 +3,102 @@ var path = require('path')
|
||||
var utils = require('./utils')
|
||||
const { uuid } = require('uuidv4');
|
||||
const config_api = require('./config');
|
||||
const { MongoClient } = require("mongodb");
|
||||
|
||||
const low = require('lowdb')
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
const local_adapter = new FileSync('./appdata/local_db.json');
|
||||
const local_db = low(local_adapter);
|
||||
|
||||
var logger = null;
|
||||
var db = null;
|
||||
var users_db = null;
|
||||
function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db }
|
||||
function setLogger(input_logger) { logger = input_logger; }
|
||||
var database = null;
|
||||
|
||||
const tables = ['files', 'playlists', 'categories', 'subscriptions', 'downloads', 'users', 'roles', 'test'];
|
||||
|
||||
const local_db_defaults = {}
|
||||
tables.forEach(table => {local_db_defaults[table] = []});
|
||||
local_db.defaults(local_db_defaults).write();
|
||||
|
||||
let using_local_db = config_api.getConfigItem('ytdl_use_local_db');
|
||||
|
||||
function setDB(input_db, input_users_db) {
|
||||
db = input_db; users_db = input_users_db;
|
||||
exports.db = input_db;
|
||||
exports.users_db = input_users_db
|
||||
}
|
||||
|
||||
function setLogger(input_logger) {
|
||||
logger = input_logger;
|
||||
}
|
||||
|
||||
exports.initialize = (input_db, input_users_db, input_logger) => {
|
||||
setDB(input_db, input_users_db);
|
||||
setLogger(input_logger);
|
||||
}
|
||||
|
||||
exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null) => {
|
||||
exports.connectToDB = async (retries = 5, no_fallback = false) => {
|
||||
if (using_local_db) return;
|
||||
const success = await exports._connectToDB();
|
||||
if (success) return true;
|
||||
|
||||
logger.warn(`MongoDB connection failed! Retrying ${retries} times...`);
|
||||
const retry_delay_ms = 2000;
|
||||
for (let i = 0; i < retries; i++) {
|
||||
const retry_succeeded = await exports._connectToDB();
|
||||
if (retry_succeeded) {
|
||||
logger.info(`Successfully connected to DB after ${i+1} attempt(s)`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (i !== retries - 1) {
|
||||
logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`);
|
||||
await utils.wait(retry_delay_ms);
|
||||
} else {
|
||||
logger.warn(`Retry ${i+1} failed.`);
|
||||
}
|
||||
}
|
||||
if (no_fallback) {
|
||||
logger.error('Failed to connect to MongoDB. Verify your connection string is valid.');
|
||||
return;
|
||||
}
|
||||
using_local_db = true;
|
||||
config_api.setConfigItem('ytdl_use_local_db', true);
|
||||
logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.');
|
||||
return true;
|
||||
}
|
||||
|
||||
exports._connectToDB = async () => {
|
||||
const uri = config_api.getConfigItem('ytdl_mongodb_connection_string'); // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb";
|
||||
const client = new MongoClient(uri, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
database = client.db('ytdl_material');
|
||||
const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name);
|
||||
|
||||
const missing_tables = tables.filter(table => !(existing_collections.includes(table)));
|
||||
missing_tables.forEach(async table => {
|
||||
await database.createCollection(table);
|
||||
})
|
||||
return true;
|
||||
} catch(err) {
|
||||
logger.error(err);
|
||||
return false;
|
||||
} finally {
|
||||
// Ensures that the client will close when you finish/error
|
||||
// await client.close();
|
||||
}
|
||||
}
|
||||
|
||||
exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => {
|
||||
let db_path = null;
|
||||
const file_id = utils.removeFileExtension(file_path);
|
||||
const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
|
||||
if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
|
||||
if (!file_object) {
|
||||
logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
|
||||
return false;
|
||||
@@ -37,23 +117,9 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
|
||||
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
|
||||
}
|
||||
|
||||
if (!sub) {
|
||||
if (multiUserMode) {
|
||||
const user_uid = multiUserMode.user;
|
||||
db_path = users_db.get('users').find({uid: user_uid}).get(`files`);
|
||||
} else {
|
||||
db_path = db.get(`files`);
|
||||
}
|
||||
} else {
|
||||
if (multiUserMode) {
|
||||
const user_uid = multiUserMode.user;
|
||||
db_path = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
|
||||
} else {
|
||||
db_path = db.get('subscriptions').find({id: sub.id}).get('videos');
|
||||
}
|
||||
}
|
||||
if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
|
||||
|
||||
const file_obj = registerFileDBManual(db_path, file_object);
|
||||
const file_obj = await registerFileDBManual(file_object);
|
||||
|
||||
// remove metadata JSON if needed
|
||||
if (!config_api.getConfigItem('ytdl_include_metadata')) {
|
||||
@@ -63,18 +129,48 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
|
||||
return file_obj;
|
||||
}
|
||||
|
||||
function registerFileDBManual(db_path, file_object) {
|
||||
exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
|
||||
if (!file_object) file_object = generateFileObject2(file_path, type);
|
||||
if (!file_object) {
|
||||
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
utils.fixVideoMetadataPerms2(file_path, type);
|
||||
|
||||
// add thumbnail path
|
||||
file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type);
|
||||
|
||||
// if category exists, only include essential info
|
||||
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
|
||||
|
||||
// modify duration
|
||||
if (cropFileSettings) {
|
||||
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
|
||||
}
|
||||
|
||||
if (user_uid) file_object['user_uid'] = user_uid;
|
||||
if (sub_id) file_object['sub_id'] = sub_id;
|
||||
|
||||
const file_obj = await registerFileDBManual(file_object);
|
||||
|
||||
// remove metadata JSON if needed
|
||||
if (!config_api.getConfigItem('ytdl_include_metadata')) {
|
||||
utils.deleteJSONFile2(file_path, type)
|
||||
}
|
||||
|
||||
return file_obj;
|
||||
}
|
||||
|
||||
async function registerFileDBManual(file_object) {
|
||||
// add additional info
|
||||
file_object['uid'] = uuid();
|
||||
file_object['registered'] = Date.now();
|
||||
path_object = path.parse(file_object['path']);
|
||||
file_object['path'] = path.format(path_object);
|
||||
|
||||
// remove duplicate(s)
|
||||
db_path.remove({path: file_object['path']}).write();
|
||||
exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
|
||||
|
||||
// add new file to db
|
||||
db_path.push(file_object).write();
|
||||
return file_object;
|
||||
}
|
||||
|
||||
@@ -107,11 +203,38 @@ function generateFileObject(id, type, customPath = null, sub = null) {
|
||||
return file_obj;
|
||||
}
|
||||
|
||||
function generateFileObject2(file_path, type) {
|
||||
var jsonobj = utils.getJSON(file_path, type);
|
||||
if (!jsonobj) {
|
||||
return null;
|
||||
}
|
||||
const ext = (type === 'audio') ? '.mp3' : '.mp4'
|
||||
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
|
||||
// console.
|
||||
var stats = fs.statSync(true_file_path);
|
||||
|
||||
const file_id = utils.removeFileExtension(path.basename(file_path));
|
||||
var title = jsonobj.title;
|
||||
var url = jsonobj.webpage_url;
|
||||
var uploader = jsonobj.uploader;
|
||||
var upload_date = jsonobj.upload_date;
|
||||
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
|
||||
|
||||
var size = stats.size;
|
||||
|
||||
var thumbnail = jsonobj.thumbnail;
|
||||
var duration = jsonobj.duration;
|
||||
var isaudio = type === 'audio';
|
||||
var description = jsonobj.description;
|
||||
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
|
||||
return file_obj;
|
||||
}
|
||||
|
||||
function getAppendedBasePathSub(sub, base_path) {
|
||||
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
|
||||
}
|
||||
|
||||
exports.getFileDirectoriesAndDBs = () => {
|
||||
exports.getFileDirectoriesAndDBs = async () => {
|
||||
let dirs_to_check = [];
|
||||
let subscriptions_to_check = [];
|
||||
const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode
|
||||
@@ -119,48 +242,45 @@ exports.getFileDirectoriesAndDBs = () => {
|
||||
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
||||
const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions');
|
||||
if (multi_user_mode) {
|
||||
let users = users_db.get('users').value();
|
||||
const users = await exports.getRecords('users');
|
||||
for (let i = 0; i < users.length; i++) {
|
||||
const user = users[i];
|
||||
|
||||
if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']);
|
||||
|
||||
// add user's audio dir to check list
|
||||
dirs_to_check.push({
|
||||
basePath: path.join(usersFileFolder, user.uid, 'audio'),
|
||||
dbPath: users_db.get('users').find({uid: user.uid}).get('files'),
|
||||
user_uid: user.uid,
|
||||
type: 'audio'
|
||||
});
|
||||
|
||||
// add user's video dir to check list
|
||||
dirs_to_check.push({
|
||||
basePath: path.join(usersFileFolder, user.uid, 'video'),
|
||||
dbPath: users_db.get('users').find({uid: user.uid}).get('files'),
|
||||
type: 'video'
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
|
||||
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
|
||||
const subscriptions = db.get('subscriptions').value();
|
||||
|
||||
if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
|
||||
|
||||
// add audio dir to check list
|
||||
dirs_to_check.push({
|
||||
basePath: audioFolderPath,
|
||||
dbPath: db.get('files'),
|
||||
type: 'audio'
|
||||
});
|
||||
|
||||
// add video dir to check list
|
||||
dirs_to_check.push({
|
||||
basePath: videoFolderPath,
|
||||
dbPath: db.get('files'),
|
||||
type: 'video'
|
||||
});
|
||||
}
|
||||
|
||||
if (subscriptions_enabled) {
|
||||
const subscriptions = await exports.getRecords('subscriptions');
|
||||
subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
|
||||
}
|
||||
|
||||
// add subscriptions to check list
|
||||
for (let i = 0; i < subscriptions_to_check.length; i++) {
|
||||
let subscription_to_check = subscriptions_to_check[i];
|
||||
@@ -169,11 +289,11 @@ exports.getFileDirectoriesAndDBs = () => {
|
||||
continue;
|
||||
}
|
||||
dirs_to_check.push({
|
||||
basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
|
||||
basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
|
||||
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
|
||||
dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos')
|
||||
: db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'),
|
||||
type: subscription_to_check.type
|
||||
user_uid: subscription_to_check.user_uid,
|
||||
type: subscription_to_check.type,
|
||||
sub_id: subscription_to_check['id']
|
||||
});
|
||||
}
|
||||
|
||||
@@ -181,22 +301,25 @@ exports.getFileDirectoriesAndDBs = () => {
|
||||
}
|
||||
|
||||
exports.importUnregisteredFiles = async () => {
|
||||
const dirs_to_check = exports.getFileDirectoriesAndDBs();
|
||||
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
|
||||
|
||||
// run through check list and check each file to see if it's missing from the db
|
||||
for (const dir_to_check of dirs_to_check) {
|
||||
for (let i = 0; i < dirs_to_check.length; i++) {
|
||||
const dir_to_check = dirs_to_check[i];
|
||||
// recursively get all files in dir's path
|
||||
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
|
||||
|
||||
files.forEach(file => {
|
||||
for (let j = 0; j < files.length; j++) {
|
||||
const file = files[j];
|
||||
|
||||
// check if file exists in db, if not add it
|
||||
const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value())
|
||||
const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id}))
|
||||
if (!file_is_registered) {
|
||||
// add additional info
|
||||
registerFileDBManual(dir_to_check.dbPath, file);
|
||||
await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
|
||||
logger.verbose(`Added discovered file to the database: ${file.id}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -204,26 +327,45 @@ exports.importUnregisteredFiles = async () => {
|
||||
exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
|
||||
const preimported_file_paths = [];
|
||||
|
||||
let dbPath = null;
|
||||
if (sub.user_uid)
|
||||
dbPath = users_db.get('users').find({uid: sub.user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
|
||||
else
|
||||
dbPath = db.get('subscriptions').find({id: sub.id}).get('videos');
|
||||
|
||||
const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
|
||||
files.forEach(file => {
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
// check if file exists in db, if not add it
|
||||
const file_is_registered = !!(dbPath.find({id: file.id}).value())
|
||||
const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
|
||||
if (!file_is_registered) {
|
||||
// add additional info
|
||||
registerFileDBManual(dbPath, file);
|
||||
await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
|
||||
preimported_file_paths.push(file['path']);
|
||||
logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
return preimported_file_paths;
|
||||
}
|
||||
|
||||
exports.addMetadataPropertyToDB = async (property_key) => {
|
||||
try {
|
||||
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
|
||||
const update_obj = {};
|
||||
for (let i = 0; i < dirs_to_check.length; i++) {
|
||||
const dir_to_check = dirs_to_check[i];
|
||||
|
||||
// recursively get all files in dir's path
|
||||
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
|
||||
for (let j = 0; j < files.length; j++) {
|
||||
const file = files[j];
|
||||
if (file[property_key]) {
|
||||
update_obj[file.uid] = {[property_key]: file[property_key]};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return await exports.bulkUpdateRecords('files', 'uid', update_obj);
|
||||
} catch(err) {
|
||||
logger.error(err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => {
|
||||
let new_playlist = {
|
||||
name: playlist_name,
|
||||
@@ -236,28 +378,19 @@ exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_u
|
||||
|
||||
const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid);
|
||||
new_playlist.duration = duration;
|
||||
|
||||
if (user_uid) {
|
||||
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
|
||||
} else {
|
||||
db.get(`playlists`)
|
||||
.push(new_playlist)
|
||||
.write();
|
||||
}
|
||||
|
||||
new_playlist.user_uid = user_uid ? user_uid : undefined;
|
||||
|
||||
await exports.insertRecordIntoTable('playlists', new_playlist);
|
||||
|
||||
return new_playlist;
|
||||
}
|
||||
|
||||
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
|
||||
let playlist = null
|
||||
if (user_uid) {
|
||||
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlist_id}).value();
|
||||
} else {
|
||||
playlist = db.get(`playlists`).find({id: playlist_id}).value();
|
||||
}
|
||||
let playlist = await exports.getRecord('playlists', {id: playlist_id});
|
||||
|
||||
if (!playlist) {
|
||||
playlist = db.get('categories').find({uid: playlist_id}).value();
|
||||
playlist = await exports.getRecord('categories', {uid: playlist_id});
|
||||
if (playlist) {
|
||||
// category found
|
||||
const files = await exports.getFiles(user_uid);
|
||||
@@ -271,7 +404,7 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
|
||||
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
|
||||
for (let i = 0; i < playlist['fileNames'].length; i++) {
|
||||
const fileName = playlist['fileNames'][i];
|
||||
const uid = exports.getVideoUIDByID(fileName, user_uid);
|
||||
const uid = await exports.getVideoUIDByID(fileName, user_uid);
|
||||
if (uid) playlist['uids'].push(uid);
|
||||
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
|
||||
}
|
||||
@@ -290,14 +423,21 @@ exports.updatePlaylist = async (playlist, user_uid = null) => {
|
||||
const duration = await exports.calculatePlaylistDuration(playlist, user_uid);
|
||||
playlist.duration = duration;
|
||||
|
||||
let db_loc = null;
|
||||
if (user_uid) {
|
||||
db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID});
|
||||
} else {
|
||||
db_loc = db.get(`playlists`).find({id: playlistID});
|
||||
return await exports.updateRecord('playlists', {id: playlistID}, playlist);
|
||||
}
|
||||
|
||||
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
|
||||
let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
|
||||
|
||||
if (!success) {
|
||||
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
|
||||
}
|
||||
db_loc.assign(playlist).write();
|
||||
return true;
|
||||
|
||||
if (!success) {
|
||||
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => {
|
||||
@@ -381,8 +521,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
|
||||
if (jsonExists) await fs.unlink(jsonPath);
|
||||
if (thumbnailExists) await fs.unlink(thumbnailPath);
|
||||
|
||||
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
|
||||
base_db_path.get('files').remove({uid: uid}).write();
|
||||
await exports.removeRecord('files', {uid: uid});
|
||||
|
||||
if (fileExists) {
|
||||
await fs.unlink(file_obj.path);
|
||||
@@ -398,29 +537,462 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
|
||||
}
|
||||
|
||||
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
|
||||
exports.getVideoUIDByID = (file_id, uuid = null) => {
|
||||
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
|
||||
const file_obj = base_db_path.get('files').find({id: file_id}).value();
|
||||
exports.getVideoUIDByID = async (file_id, uuid = null) => {
|
||||
const file_obj = await exports.getRecord('files', {id: file_id});
|
||||
return file_obj ? file_obj['uid'] : null;
|
||||
}
|
||||
|
||||
exports.getVideo = async (file_uid, uuid = null, sub_id = null) => {
|
||||
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
|
||||
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
|
||||
return sub_db_path.find({uid: file_uid}).value();
|
||||
return await exports.getRecord('files', {uid: file_uid});
|
||||
}
|
||||
|
||||
exports.getFiles = async (uuid = null) => {
|
||||
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
|
||||
return base_db_path.get('files').value();
|
||||
return await exports.getRecords('files', {user_uid: uuid});
|
||||
}
|
||||
|
||||
exports.setVideoProperty = async (file_uid, assignment_obj, uuid, sub_id) => {
|
||||
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
|
||||
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
|
||||
const file_db_path = sub_db_path.find({uid: file_uid});
|
||||
if (!(file_db_path.value())) {
|
||||
logger.error(`Failed to find file with uid ${file_uid}`);
|
||||
}
|
||||
sub_db_path.find({uid: file_uid}).assign(assignment_obj).write();
|
||||
exports.setVideoProperty = async (file_uid, assignment_obj) => {
|
||||
// TODO: check if video exists, throw error if not
|
||||
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
|
||||
}
|
||||
|
||||
// DB to JSON
|
||||
|
||||
exports.exportDBToJSON = async (tables) => {
|
||||
const users_db_json = await createUsersJSONs(tables.files, tables.playlists, tables.subscriptions, tables.categories, tables.users);
|
||||
const db_json = await createNonUserJSON(tables.files, tables.playlists, tables.subscriptions, tables.categories);
|
||||
|
||||
return {users_db_json: users_db_json, db_json: db_json};
|
||||
}
|
||||
|
||||
const createUsersJSONs = async (files, playlists, subscriptions, categories, users) => {
|
||||
// we need to already have a list of user objects to gather the records into
|
||||
for (let user of users) {
|
||||
const files_of_user = files.filter(file => file.user_uid === user.uid && !file.sub_id);
|
||||
const playlists_of_user = playlists.filter(playlist => playlist.user_uid === user.uid);
|
||||
const subs_of_user = subscriptions.filter(sub => sub.user_uid === user.uid);
|
||||
const categories_of_user = categories ? categories.filter(category => category && category.user_uid === user.uid) : [];
|
||||
user['files'] = files_of_user;
|
||||
user['playlists'] = playlists_of_user;
|
||||
user['subscriptions'] = subs_of_user;
|
||||
user['categories'] = categories_of_user;
|
||||
|
||||
for (let subscription of subscriptions) {
|
||||
subscription['videos'] = files.filter(file => file.user_uid === user.uid && file.sub_id === sub.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const createNonUserJSON = async (files, playlists, subscriptions, categories) => {
|
||||
const non_user_json = {
|
||||
files: files.filter(file => !file.user_uid && !file.sub_id),
|
||||
playlists: playlists.filter(playlist => !playlist.user_uid),
|
||||
subscriptions: subscriptions.filter(sub => !sub.user_uid),
|
||||
categories: categories ? categories.filter(category => category && !category.user_uid) : []
|
||||
}
|
||||
|
||||
for (let subscription of non_user_json['subscriptions']) {
|
||||
subscription['videos'] = files.filter(file => !file.user_uid && file.sub_id === subscription.id);
|
||||
}
|
||||
|
||||
return non_user_json;
|
||||
}
|
||||
|
||||
// Basic DB functions
|
||||
|
||||
// Create
|
||||
|
||||
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
|
||||
local_db.get(table).push(doc).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
|
||||
|
||||
const output = await database.collection(table).insertOne(doc);
|
||||
logger.debug(`Inserted doc into ${table}`);
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.insertRecordsIntoTable = async (table, docs) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
local_db.get(table).push(...docs).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const output = await database.collection(table).insertMany(docs);
|
||||
logger.debug(`Inserted ${output.insertedCount} docs into ${table}`);
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.bulkInsertRecordsIntoTable = async (table, docs) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
return await exports.insertRecordsIntoTable(table, docs);
|
||||
}
|
||||
|
||||
// not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed
|
||||
const table_collection = database.collection(table);
|
||||
|
||||
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
|
||||
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
bulk.insert(docs[i]);
|
||||
}
|
||||
|
||||
const output = await bulk.execute();
|
||||
return !!(output['result']['ok']);
|
||||
|
||||
}
|
||||
|
||||
// Read
|
||||
|
||||
exports.getRecord = async (table, filter_obj) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
|
||||
}
|
||||
|
||||
return await database.collection(table).findOne(filter_obj);
|
||||
}
|
||||
|
||||
exports.getRecords = async (table, filter_obj = null) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
|
||||
}
|
||||
|
||||
return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray();
|
||||
}
|
||||
|
||||
// Update
|
||||
|
||||
exports.updateRecord = async (table, filter_obj, update_obj) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
// sometimes _id will be in the update obj, this breaks mongodb
|
||||
if (update_obj['_id']) delete update_obj['_id'];
|
||||
const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj});
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.updateRecords = async (table, filter_obj, update_obj) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj});
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
local_db.get(table).each((record) => {
|
||||
const item_id_to_update = record[key_label];
|
||||
if (!update_obj[item_id_to_update]) return;
|
||||
|
||||
const props_to_update = Object.keys(update_obj[item_id_to_update]);
|
||||
for (let i = 0; i < props_to_update.length; i++) {
|
||||
const prop_to_update = props_to_update[i];
|
||||
const prop_value = update_obj[item_id_to_update][prop_to_update];
|
||||
record[prop_to_update] = prop_value;
|
||||
}
|
||||
}).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const table_collection = database.collection(table);
|
||||
|
||||
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
|
||||
|
||||
const item_ids_to_update = Object.keys(update_obj);
|
||||
|
||||
for (let i = 0; i < item_ids_to_update.length; i++) {
|
||||
const item_id_to_update = item_ids_to_update[i];
|
||||
bulk.find({[key_label]: item_id_to_update }).updateOne({
|
||||
"$set": update_obj[item_id_to_update]
|
||||
});
|
||||
}
|
||||
|
||||
const output = await bulk.execute();
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}});
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}});
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
// Delete
|
||||
|
||||
exports.removeRecord = async (table, filter_obj) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
|
||||
return true;
|
||||
}
|
||||
|
||||
const output = await database.collection(table).deleteOne(filter_obj);
|
||||
return !!(output['result']['ok']);
|
||||
}
|
||||
|
||||
exports.removeAllRecords = async (table = null) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
const tables_to_remove = table ? [table] : tables;
|
||||
logger.debug(`Removing all records from: ${tables_to_remove}`)
|
||||
for (let i = 0; i < tables_to_remove.length; i++) {
|
||||
const table_to_remove = tables_to_remove[i];
|
||||
local_db.assign({[table_to_remove]: []}).write();
|
||||
logger.debug(`Removed all records from ${table_to_remove}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
let success = true;
|
||||
const tables_to_remove = table ? [table] : tables;
|
||||
logger.debug(`Removing all records from: ${tables_to_remove}`)
|
||||
for (let i = 0; i < tables_to_remove.length; i++) {
|
||||
const table_to_remove = tables_to_remove[i];
|
||||
|
||||
const output = await database.collection(table_to_remove).deleteMany({});
|
||||
logger.debug(`Removed all records from ${table_to_remove}`);
|
||||
success &= !!(output['result']['ok']);
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
// Stats
|
||||
|
||||
exports.getDBStats = async () => {
|
||||
const stats_by_table = {};
|
||||
for (let i = 0; i < tables.length; i++) {
|
||||
const table = tables[i];
|
||||
if (table === 'test') continue;
|
||||
|
||||
stats_by_table[table] = await getDBTableStats(table);
|
||||
}
|
||||
return {stats_by_table: stats_by_table, using_local_db: using_local_db};
|
||||
}
|
||||
|
||||
const getDBTableStats = async (table) => {
|
||||
const table_stats = {};
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
table_stats['records_count'] = local_db.get(table).value().length;
|
||||
} else {
|
||||
const stats = await database.collection(table).stats();
|
||||
table_stats['records_count'] = stats.count;
|
||||
}
|
||||
return table_stats;
|
||||
}
|
||||
|
||||
// JSON to DB
|
||||
|
||||
exports.generateJSONTables = async (db_json, users_json) => {
|
||||
// create records
|
||||
let files = db_json['files'] || [];
|
||||
let playlists = db_json['playlists'] || [];
|
||||
let categories = db_json['categories'] || [];
|
||||
let subscriptions = db_json['subscriptions'] || [];
|
||||
|
||||
const users = users_json['users'];
|
||||
|
||||
for (let i = 0; i < users.length; i++) {
|
||||
const user = users[i];
|
||||
|
||||
if (user['files']) {
|
||||
user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] }));
|
||||
files = files.concat(user['files']);
|
||||
}
|
||||
if (user['playlists']) {
|
||||
user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] }));
|
||||
playlists = playlists.concat(user['playlists']);
|
||||
}
|
||||
if (user['categories']) {
|
||||
user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] }));
|
||||
categories = categories.concat(user['categories']);
|
||||
}
|
||||
|
||||
if (user['subscriptions']) {
|
||||
user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] }));
|
||||
subscriptions = subscriptions.concat(user['subscriptions']);
|
||||
}
|
||||
}
|
||||
|
||||
const tables_obj = {};
|
||||
|
||||
// TODO: use create*Records funcs to strip unnecessary properties
|
||||
tables_obj.files = createFilesRecords(files, subscriptions);
|
||||
tables_obj.playlists = playlists;
|
||||
tables_obj.categories = categories;
|
||||
tables_obj.subscriptions = createSubscriptionsRecords(subscriptions);
|
||||
tables_obj.users = createUsersRecords(users);
|
||||
tables_obj.roles = createRolesRecords(users_json['roles']);
|
||||
tables_obj.downloads = createDownloadsRecords(db_json['downloads'])
|
||||
|
||||
return tables_obj;
|
||||
}
|
||||
|
||||
exports.importJSONToDB = async (db_json, users_json) => {
|
||||
// TODO: backup db
|
||||
|
||||
// TODO: delete current records
|
||||
const tables_obj = await exports.generateJSONTables(db_json, users_json);
|
||||
|
||||
const table_keys = Object.keys(tables_obj);
|
||||
|
||||
let success = true;
|
||||
for (let i = 0; i < table_keys.length; i++) {
|
||||
const table_key = table_keys[i];
|
||||
success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key]);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
const createFilesRecords = (files, subscriptions) => {
|
||||
for (let i = 0; i < subscriptions.length; i++) {
|
||||
const subscription = subscriptions[i];
|
||||
subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
|
||||
files = files.concat(subscriptions[i]['videos']);
|
||||
console.log(files.length);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
const createPlaylistsRecords = async (playlists) => {
|
||||
|
||||
}
|
||||
|
||||
const createCategoriesRecords = async (categories) => {
|
||||
|
||||
}
|
||||
|
||||
const createSubscriptionsRecords = (subscriptions) => {
|
||||
for (let i = 0; i < subscriptions.length; i++) {
|
||||
delete subscriptions[i]['videos'];
|
||||
}
|
||||
|
||||
return subscriptions;
|
||||
}
|
||||
|
||||
const createUsersRecords = (users) => {
|
||||
users.forEach(user => {
|
||||
delete user['files'];
|
||||
delete user['playlists'];
|
||||
delete user['subscriptions'];
|
||||
});
|
||||
return users;
|
||||
}
|
||||
|
||||
const createRolesRecords = (roles) => {
|
||||
const new_roles = [];
|
||||
Object.keys(roles).forEach(role_key => {
|
||||
new_roles.push({
|
||||
key: role_key,
|
||||
...roles[role_key]
|
||||
});
|
||||
});
|
||||
return new_roles;
|
||||
}
|
||||
|
||||
const createDownloadsRecords = (downloads) => {
|
||||
const new_downloads = [];
|
||||
Object.keys(downloads).forEach(session_key => {
|
||||
new_downloads.push({
|
||||
key: session_key,
|
||||
...downloads[session_key]
|
||||
});
|
||||
});
|
||||
return new_downloads;
|
||||
}
|
||||
|
||||
exports.transferDB = async (local_to_remote) => {
|
||||
const table_to_records = {};
|
||||
for (let i = 0; i < tables.length; i++) {
|
||||
const table = tables[i];
|
||||
table_to_records[table] = await exports.getRecords(table);
|
||||
}
|
||||
|
||||
using_local_db = !local_to_remote;
|
||||
if (local_to_remote) {
|
||||
// backup local DB
|
||||
logger.debug('Backup up Local DB...');
|
||||
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
|
||||
const db_connected = await exports.connectToDB(5, true);
|
||||
if (!db_connected) {
|
||||
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
|
||||
logger.debug('Clearing new database before transfer...');
|
||||
|
||||
await exports.removeAllRecords();
|
||||
|
||||
logger.debug('Database cleared! Beginning transfer.');
|
||||
|
||||
for (let i = 0; i < tables.length; i++) {
|
||||
const table = tables[i];
|
||||
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
|
||||
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
|
||||
}
|
||||
|
||||
config_api.setConfigItem('ytdl_use_local_db', using_local_db);
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
/*
|
||||
This function is necessary to emulate mongodb's ability to search for null or missing values.
|
||||
A filter of null or undefined for a property will find docs that have that property missing, or have it
|
||||
null or undefined. We want that same functionality for the local DB as well
|
||||
*/
|
||||
const applyFilterLocalDB = (db_path, filter_obj, operation) => {
|
||||
const filter_props = Object.keys(filter_obj);
|
||||
const return_val = db_path[operation](record => {
|
||||
if (!filter_props) return true;
|
||||
let filtered = true;
|
||||
for (let i = 0; i < filter_props.length; i++) {
|
||||
const filter_prop = filter_props[i];
|
||||
const filter_prop_value = filter_obj[filter_prop];
|
||||
if (filter_prop_value === undefined || filter_prop_value === null) {
|
||||
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
|
||||
} else {
|
||||
filtered &= record[filter_prop] === filter_prop_value;
|
||||
}
|
||||
}
|
||||
return filtered;
|
||||
});
|
||||
return return_val;
|
||||
}
|
||||
Reference in New Issue
Block a user