mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-07 20:10:03 +03:00
ca5381fe0f
Added DB functionality to remove properties from records DB records in local DB can now be updated if nested
1183 lines
42 KiB
JavaScript
1183 lines
42 KiB
JavaScript
var fs = require('fs-extra')
|
|
var path = require('path')
|
|
const { MongoClient } = require("mongodb");
|
|
const { uuid } = require('uuidv4');
|
|
const _ = require('lodash');
|
|
|
|
const config_api = require('./config');
|
|
var utils = require('./utils')
|
|
const logger = require('./logger');
|
|
|
|
const low = require('lowdb')
|
|
const FileSync = require('lowdb/adapters/FileSync');
|
|
const { BehaviorSubject } = require('rxjs');
|
|
const local_adapter = new FileSync('./appdata/local_db.json');
|
|
const local_db = low(local_adapter);
|
|
|
|
let database = null;
|
|
exports.database_initialized = false;
|
|
exports.database_initialized_bs = new BehaviorSubject(false);
|
|
|
|
const tables = {
|
|
files: {
|
|
name: 'files',
|
|
primary_key: 'uid',
|
|
text_search: {
|
|
title: 'text',
|
|
uploader: 'text',
|
|
uid: 'text'
|
|
}
|
|
},
|
|
playlists: {
|
|
name: 'playlists',
|
|
primary_key: 'id'
|
|
},
|
|
categories: {
|
|
name: 'categories',
|
|
primary_key: 'uid'
|
|
},
|
|
subscriptions: {
|
|
name: 'subscriptions',
|
|
primary_key: 'id'
|
|
},
|
|
downloads: {
|
|
name: 'downloads'
|
|
},
|
|
users: {
|
|
name: 'users',
|
|
primary_key: 'uid'
|
|
},
|
|
roles: {
|
|
name: 'roles',
|
|
primary_key: 'key'
|
|
},
|
|
download_queue: {
|
|
name: 'download_queue',
|
|
primary_key: 'uid'
|
|
},
|
|
tasks: {
|
|
name: 'tasks',
|
|
primary_key: 'key'
|
|
},
|
|
notifications: {
|
|
name: 'notifications',
|
|
primary_key: 'uid'
|
|
},
|
|
archives: {
|
|
name: 'archives'
|
|
},
|
|
test: {
|
|
name: 'test'
|
|
}
|
|
}
|
|
|
|
const tables_list = Object.keys(tables);
|
|
|
|
const local_db_defaults = {}
|
|
tables_list.forEach(table => {local_db_defaults[table] = []});
|
|
local_db.defaults(local_db_defaults).write();
|
|
|
|
let using_local_db = null;
|
|
|
|
function setDB(input_db, input_users_db) {
|
|
db = input_db; users_db = input_users_db;
|
|
exports.db = input_db;
|
|
exports.users_db = input_users_db
|
|
}
|
|
|
|
exports.initialize = (input_db, input_users_db) => {
|
|
setDB(input_db, input_users_db);
|
|
|
|
// must be done here to prevent getConfigItem from being called before init
|
|
using_local_db = config_api.getConfigItem('ytdl_use_local_db');
|
|
}
|
|
|
|
exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
|
|
const success = await exports._connectToDB(custom_connection_string);
|
|
if (success) return true;
|
|
|
|
if (retries) {
|
|
logger.warn(`MongoDB connection failed! Retrying ${retries} times...`);
|
|
const retry_delay_ms = 2000;
|
|
for (let i = 0; i < retries; i++) {
|
|
const retry_succeeded = await exports._connectToDB();
|
|
if (retry_succeeded) {
|
|
logger.info(`Successfully connected to DB after ${i+1} attempt(s)`);
|
|
return true;
|
|
}
|
|
|
|
if (i !== retries - 1) {
|
|
logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`);
|
|
await utils.wait(retry_delay_ms);
|
|
} else {
|
|
logger.warn(`Retry ${i+1} failed.`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (no_fallback) {
|
|
logger.error('Failed to connect to MongoDB. Verify your connection string is valid.');
|
|
return;
|
|
}
|
|
using_local_db = true;
|
|
config_api.setConfigItem('ytdl_use_local_db', true);
|
|
logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.');
|
|
return true;
|
|
}
|
|
|
|
exports._connectToDB = async (custom_connection_string = null) => {
|
|
const uri = !custom_connection_string ? config_api.getConfigItem('ytdl_mongodb_connection_string') : custom_connection_string; // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb";
|
|
const client = new MongoClient(uri, {
|
|
useNewUrlParser: true,
|
|
useUnifiedTopology: true,
|
|
});
|
|
|
|
try {
|
|
await client.connect();
|
|
database = client.db('ytdl_material');
|
|
|
|
// avoid doing anything else if it's just a test
|
|
if (custom_connection_string) return true;
|
|
|
|
const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name);
|
|
|
|
const missing_tables = tables_list.filter(table => !(existing_collections.includes(table)));
|
|
missing_tables.forEach(async table => {
|
|
await database.createCollection(table);
|
|
});
|
|
|
|
tables_list.forEach(async table => {
|
|
const primary_key = tables[table]['primary_key'];
|
|
if (primary_key) {
|
|
await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
|
|
}
|
|
const text_search = tables[table]['text_search'];
|
|
if (text_search) {
|
|
await database.collection(table).createIndex(text_search);
|
|
}
|
|
});
|
|
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
|
|
return true;
|
|
} catch(err) {
|
|
logger.error(err);
|
|
return false;
|
|
} finally {
|
|
// Ensures that the client will close when you finish/error
|
|
// await client.close();
|
|
}
|
|
}
|
|
|
|
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
|
|
if (!file_object) file_object = generateFileObject(file_path, type);
|
|
if (!file_object) {
|
|
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
|
|
return false;
|
|
}
|
|
|
|
utils.fixVideoMetadataPerms(file_path, type);
|
|
|
|
// add thumbnail path
|
|
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
|
|
|
|
// if category exists, only include essential info
|
|
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
|
|
|
|
// modify duration
|
|
if (cropFileSettings) {
|
|
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
|
|
}
|
|
|
|
if (user_uid) file_object['user_uid'] = user_uid;
|
|
if (sub_id) file_object['sub_id'] = sub_id;
|
|
|
|
const file_obj = await registerFileDBManual(file_object);
|
|
|
|
// remove metadata JSON if needed
|
|
if (!config_api.getConfigItem('ytdl_include_metadata')) {
|
|
utils.deleteJSONFile(file_path, type)
|
|
}
|
|
|
|
return file_obj;
|
|
}
|
|
|
|
async function registerFileDBManual(file_object) {
|
|
// add additional info
|
|
file_object['uid'] = uuid();
|
|
file_object['registered'] = Date.now();
|
|
path_object = path.parse(file_object['path']);
|
|
file_object['path'] = path.format(path_object);
|
|
|
|
await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
|
|
|
|
return file_object;
|
|
}
|
|
|
|
function generateFileObject(file_path, type) {
|
|
var jsonobj = utils.getJSON(file_path, type);
|
|
if (!jsonobj) {
|
|
return null;
|
|
} else if (!jsonobj['_filename']) {
|
|
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
|
|
return null;
|
|
}
|
|
const ext = (type === 'audio') ? '.mp3' : '.mp4'
|
|
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
|
|
// console.
|
|
var stats = fs.statSync(true_file_path);
|
|
|
|
const file_id = utils.removeFileExtension(path.basename(file_path));
|
|
var title = jsonobj.title;
|
|
var url = jsonobj.webpage_url;
|
|
var uploader = jsonobj.uploader;
|
|
var upload_date = utils.formatDateString(jsonobj.upload_date);
|
|
|
|
var size = stats.size;
|
|
|
|
var thumbnail = jsonobj.thumbnail;
|
|
var duration = jsonobj.duration;
|
|
var isaudio = type === 'audio';
|
|
var description = jsonobj.description;
|
|
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
|
|
return file_obj;
|
|
}
|
|
|
|
function getAppendedBasePathSub(sub, base_path) {
|
|
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
|
|
}
|
|
|
|
exports.getFileDirectoriesAndDBs = async () => {
|
|
let dirs_to_check = [];
|
|
let subscriptions_to_check = [];
|
|
const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode
|
|
const multi_user_mode = config_api.getConfigItem('ytdl_multi_user_mode');
|
|
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
|
const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions');
|
|
if (multi_user_mode) {
|
|
const users = await exports.getRecords('users');
|
|
for (let i = 0; i < users.length; i++) {
|
|
const user = users[i];
|
|
|
|
// add user's audio dir to check list
|
|
dirs_to_check.push({
|
|
basePath: path.join(usersFileFolder, user.uid, 'audio'),
|
|
user_uid: user.uid,
|
|
type: 'audio',
|
|
archive_path: utils.getArchiveFolder('audio', user.uid)
|
|
});
|
|
|
|
// add user's video dir to check list
|
|
dirs_to_check.push({
|
|
basePath: path.join(usersFileFolder, user.uid, 'video'),
|
|
user_uid: user.uid,
|
|
type: 'video',
|
|
archive_path: utils.getArchiveFolder('video', user.uid)
|
|
});
|
|
}
|
|
} else {
|
|
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
|
|
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
|
|
|
|
// add audio dir to check list
|
|
dirs_to_check.push({
|
|
basePath: audioFolderPath,
|
|
type: 'audio',
|
|
archive_path: utils.getArchiveFolder('audio')
|
|
});
|
|
|
|
// add video dir to check list
|
|
dirs_to_check.push({
|
|
basePath: videoFolderPath,
|
|
type: 'video',
|
|
archive_path: utils.getArchiveFolder('video')
|
|
});
|
|
}
|
|
|
|
if (subscriptions_enabled) {
|
|
const subscriptions = await exports.getRecords('subscriptions');
|
|
subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
|
|
}
|
|
|
|
// add subscriptions to check list
|
|
for (let i = 0; i < subscriptions_to_check.length; i++) {
|
|
let subscription_to_check = subscriptions_to_check[i];
|
|
if (!subscription_to_check.name) {
|
|
// TODO: Remove subscription as it'll never complete
|
|
continue;
|
|
}
|
|
dirs_to_check.push({
|
|
basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
|
|
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
|
|
user_uid: subscription_to_check.user_uid,
|
|
type: subscription_to_check.type,
|
|
sub_id: subscription_to_check['id'],
|
|
archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
|
|
});
|
|
}
|
|
|
|
return dirs_to_check;
|
|
}
|
|
|
|
exports.importUnregisteredFiles = async () => {
|
|
const imported_files = [];
|
|
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
|
|
|
|
// run through check list and check each file to see if it's missing from the db
|
|
for (let i = 0; i < dirs_to_check.length; i++) {
|
|
const dir_to_check = dirs_to_check[i];
|
|
// recursively get all files in dir's path
|
|
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
|
|
|
|
for (let j = 0; j < files.length; j++) {
|
|
const file = files[j];
|
|
|
|
// check if file exists in db, if not add it
|
|
const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
|
|
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
|
|
if (!file_is_registered) {
|
|
// add additional info
|
|
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
|
|
if (file_obj) {
|
|
imported_files.push(file_obj['uid']);
|
|
logger.verbose(`Added discovered file to the database: ${file.id}`);
|
|
} else {
|
|
logger.error(`Failed to import ${file['path']} automatically.`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return imported_files;
|
|
}
|
|
|
|
exports.addMetadataPropertyToDB = async (property_key) => {
|
|
try {
|
|
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
|
|
const update_obj = {};
|
|
for (let i = 0; i < dirs_to_check.length; i++) {
|
|
const dir_to_check = dirs_to_check[i];
|
|
|
|
// recursively get all files in dir's path
|
|
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
|
|
for (let j = 0; j < files.length; j++) {
|
|
const file = files[j];
|
|
if (file[property_key]) {
|
|
update_obj[file.uid] = {[property_key]: file[property_key]};
|
|
}
|
|
}
|
|
}
|
|
|
|
return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
|
|
} catch(err) {
|
|
logger.error(err);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
|
|
const first_video = await exports.getVideo(uids[0]);
|
|
const thumbnailToUse = first_video['thumbnailURL'];
|
|
|
|
let new_playlist = {
|
|
name: playlist_name,
|
|
uids: uids,
|
|
id: uuid(),
|
|
thumbnailURL: thumbnailToUse,
|
|
registered: Date.now(),
|
|
randomize_order: false
|
|
};
|
|
|
|
new_playlist.user_uid = user_uid ? user_uid : undefined;
|
|
|
|
await exports.insertRecordIntoTable('playlists', new_playlist);
|
|
|
|
const duration = await exports.calculatePlaylistDuration(new_playlist);
|
|
await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
|
|
|
|
return new_playlist;
|
|
}
|
|
|
|
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
|
|
let playlist = await exports.getRecord('playlists', {id: playlist_id});
|
|
|
|
if (!playlist) {
|
|
playlist = await exports.getRecord('categories', {uid: playlist_id});
|
|
if (playlist) {
|
|
const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
|
|
playlist['uids'] = uids;
|
|
playlist['auto'] = true;
|
|
}
|
|
}
|
|
|
|
// converts playlists to new UID-based schema
|
|
if (playlist && playlist['fileNames'] && !playlist['uids']) {
|
|
playlist['uids'] = [];
|
|
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
|
|
for (let i = 0; i < playlist['fileNames'].length; i++) {
|
|
const fileName = playlist['fileNames'][i];
|
|
const uid = await exports.getVideoUIDByID(fileName, user_uid);
|
|
if (uid) playlist['uids'].push(uid);
|
|
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
|
|
}
|
|
exports.updatePlaylist(playlist, user_uid);
|
|
}
|
|
|
|
// prevent unauthorized users from accessing the file info
|
|
if (require_sharing && !playlist['sharingEnabled']) return null;
|
|
|
|
return playlist;
|
|
}
|
|
|
|
exports.updatePlaylist = async (playlist) => {
|
|
let playlistID = playlist.id;
|
|
|
|
const duration = await exports.calculatePlaylistDuration(playlist);
|
|
playlist.duration = duration;
|
|
|
|
return await exports.updateRecord('playlists', {id: playlistID}, playlist);
|
|
}
|
|
|
|
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
|
|
let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
|
|
|
|
if (!success) {
|
|
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
|
|
}
|
|
|
|
if (!success) {
|
|
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
|
|
if (!playlist_file_objs) {
|
|
playlist_file_objs = [];
|
|
for (let i = 0; i < playlist['uids'].length; i++) {
|
|
const uid = playlist['uids'][i];
|
|
const file_obj = await exports.getVideo(uid);
|
|
if (file_obj) playlist_file_objs.push(file_obj);
|
|
}
|
|
}
|
|
|
|
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
|
|
}
|
|
|
|
exports.deleteFile = async (uid, blacklistMode = false) => {
|
|
const file_obj = await exports.getVideo(uid);
|
|
const type = file_obj.isAudio ? 'audio' : 'video';
|
|
const folderPath = path.dirname(file_obj.path);
|
|
const ext = type === 'audio' ? 'mp3' : 'mp4';
|
|
const name = file_obj.id;
|
|
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
|
|
|
|
var jsonPath = `${file_obj.path}.info.json`;
|
|
var altJSONPath = `${filePathNoExtension}.info.json`;
|
|
var thumbnailPath = `${filePathNoExtension}.webp`;
|
|
var altThumbnailPath = `${filePathNoExtension}.jpg`;
|
|
|
|
jsonPath = path.join(__dirname, jsonPath);
|
|
altJSONPath = path.join(__dirname, altJSONPath);
|
|
|
|
let jsonExists = await fs.pathExists(jsonPath);
|
|
let thumbnailExists = await fs.pathExists(thumbnailPath);
|
|
|
|
if (!jsonExists) {
|
|
if (await fs.pathExists(altJSONPath)) {
|
|
jsonExists = true;
|
|
jsonPath = altJSONPath;
|
|
}
|
|
}
|
|
|
|
if (!thumbnailExists) {
|
|
if (await fs.pathExists(altThumbnailPath)) {
|
|
thumbnailExists = true;
|
|
thumbnailPath = altThumbnailPath;
|
|
}
|
|
}
|
|
|
|
let fileExists = await fs.pathExists(file_obj.path);
|
|
|
|
if (config_api.descriptors[uid]) {
|
|
try {
|
|
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
|
|
config_api.descriptors[uid][i].destroy();
|
|
}
|
|
} catch(e) {
|
|
|
|
}
|
|
}
|
|
|
|
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
|
|
if (useYoutubeDLArchive) {
|
|
// get id/extractor from JSON
|
|
|
|
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
|
|
let retrievedID = null;
|
|
let retrievedExtractor = null;
|
|
if (info_json) {
|
|
retrievedID = info_json['id'];
|
|
retrievedExtractor = info_json['extractor'];
|
|
}
|
|
|
|
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
|
|
if (!blacklistMode) {
|
|
// workaround until a files_api is created (using archive_api would make a circular dependency)
|
|
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
|
|
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
|
|
}
|
|
}
|
|
|
|
if (jsonExists) await fs.unlink(jsonPath);
|
|
if (thumbnailExists) await fs.unlink(thumbnailPath);
|
|
|
|
await exports.removeRecord('files', {uid: uid});
|
|
|
|
if (fileExists) {
|
|
await fs.unlink(file_obj.path);
|
|
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
|
|
return false;
|
|
} else {
|
|
return true;
|
|
}
|
|
} else {
|
|
// TODO: tell user that the file didn't exist
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
|
|
exports.getVideoUIDByID = async (file_id, uuid = null) => {
|
|
const file_obj = await exports.getRecord('files', {id: file_id});
|
|
return file_obj ? file_obj['uid'] : null;
|
|
}
|
|
|
|
exports.getVideo = async (file_uid) => {
|
|
return await exports.getRecord('files', {uid: file_uid});
|
|
}
|
|
|
|
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
|
|
const filter_obj = {user_uid: uuid};
|
|
const regex = true;
|
|
if (text_search) {
|
|
if (regex) {
|
|
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
|
|
} else {
|
|
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
|
|
}
|
|
}
|
|
|
|
if (favorite_filter) {
|
|
filter_obj['favorite'] = true;
|
|
}
|
|
|
|
if (sub_id) {
|
|
filter_obj['sub_id'] = sub_id;
|
|
}
|
|
|
|
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
|
|
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
|
|
|
|
const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
|
|
const file_count = await exports.getRecords('files', filter_obj, true);
|
|
|
|
return {files, file_count};
|
|
}
|
|
|
|
exports.setVideoProperty = async (file_uid, assignment_obj) => {
|
|
// TODO: check if video exists, throw error if not
|
|
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
|
|
}
|
|
|
|
// Basic DB functions
|
|
|
|
// Create
|
|
|
|
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
|
|
local_db.get(table).push(doc).write();
|
|
return true;
|
|
}
|
|
|
|
if (replaceFilter) {
|
|
const output = await database.collection(table).bulkWrite([
|
|
{
|
|
deleteMany: {
|
|
filter: replaceFilter
|
|
}
|
|
},
|
|
{
|
|
insertOne: {
|
|
document: doc
|
|
}
|
|
}
|
|
]);
|
|
logger.debug(`Inserted doc into ${table} with filter: ${JSON.stringify(replaceFilter)}`);
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
const output = await database.collection(table).insertOne(doc);
|
|
logger.debug(`Inserted doc into ${table}`);
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.insertRecordsIntoTable = async (table, docs, ignore_errors = false) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
const records_limit = 30000;
|
|
if (docs.length < records_limit) {
|
|
local_db.get(table).push(...docs).write();
|
|
} else {
|
|
for (let i = 0; i < docs.length; i+=records_limit) {
|
|
const records_to_push = docs.slice(i, i+records_limit > docs.length ? docs.length : i+records_limit)
|
|
local_db.get(table).push(...records_to_push).write();
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
const output = await database.collection(table).insertMany(docs, {ordered: !ignore_errors});
|
|
logger.debug(`Inserted ${output.insertedCount} docs into ${table}`);
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.bulkInsertRecordsIntoTable = async (table, docs) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
return await exports.insertRecordsIntoTable(table, docs);
|
|
}
|
|
|
|
// not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed
|
|
const table_collection = database.collection(table);
|
|
|
|
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
|
|
|
|
for (let i = 0; i < docs.length; i++) {
|
|
bulk.insert(docs[i]);
|
|
}
|
|
|
|
const output = await bulk.execute();
|
|
return !!(output['result']['ok']);
|
|
|
|
}
|
|
|
|
// Read
|
|
|
|
exports.getRecord = async (table, filter_obj) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
return exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
|
|
}
|
|
|
|
return await database.collection(table).findOne(filter_obj);
|
|
}
|
|
|
|
exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
let cursor = filter_obj ? exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
|
|
if (sort) {
|
|
cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
|
|
}
|
|
if (range) {
|
|
cursor = cursor.slice(range[0], range[1]);
|
|
}
|
|
return !return_count ? cursor : cursor.length;
|
|
}
|
|
|
|
const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
|
|
if (sort) {
|
|
cursor.sort({[sort['by']]: sort['order']});
|
|
}
|
|
if (range) {
|
|
cursor.skip(range[0]).limit(range[1] - range[0]);
|
|
}
|
|
|
|
return !return_count ? await cursor.toArray() : await cursor.count();
|
|
}
|
|
|
|
// Update
|
|
|
|
exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
if (nested_mode) {
|
|
// if object is nested we need to handle it differently
|
|
update_obj = utils.convertFlatObjectToNestedObject(update_obj);
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
|
|
return true;
|
|
}
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
|
|
return true;
|
|
}
|
|
|
|
// sometimes _id will be in the update obj, this breaks mongodb
|
|
if (update_obj['_id']) delete update_obj['_id'];
|
|
const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj});
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.updateRecords = async (table, filter_obj, update_obj) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
|
|
return true;
|
|
}
|
|
|
|
const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj});
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
const props_to_remove = Object.keys(remove_obj);
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
|
|
return true;
|
|
}
|
|
|
|
const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
local_db.get(table).each((record) => {
|
|
const item_id_to_update = record[key_label];
|
|
if (!update_obj[item_id_to_update]) return;
|
|
|
|
const props_to_update = Object.keys(update_obj[item_id_to_update]);
|
|
for (let i = 0; i < props_to_update.length; i++) {
|
|
const prop_to_update = props_to_update[i];
|
|
const prop_value = update_obj[item_id_to_update][prop_to_update];
|
|
record[prop_to_update] = prop_value;
|
|
}
|
|
}).write();
|
|
return true;
|
|
}
|
|
|
|
const table_collection = database.collection(table);
|
|
|
|
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
|
|
|
|
const item_ids_to_update = Object.keys(update_obj);
|
|
|
|
for (let i = 0; i < item_ids_to_update.length; i++) {
|
|
const item_id_to_update = item_ids_to_update[i];
|
|
bulk.find({[key_label]: item_id_to_update }).updateOne({
|
|
"$set": update_obj[item_id_to_update]
|
|
});
|
|
}
|
|
|
|
const output = await bulk.execute();
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
|
|
return true;
|
|
}
|
|
|
|
const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}});
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
|
|
return true;
|
|
}
|
|
|
|
const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}});
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
// Delete
|
|
|
|
exports.removeRecord = async (table, filter_obj) => {
|
|
// local db override
|
|
if (using_local_db) {
|
|
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
|
|
return true;
|
|
}
|
|
|
|
const output = await database.collection(table).deleteOne(filter_obj);
|
|
return !!(output['result']['ok']);
|
|
}
|
|
|
|
// exports.removeRecordsByUIDBulk = async (table, uids) => {
|
|
// // local db override
|
|
// if (using_local_db) {
|
|
// exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
|
|
// return true;
|
|
// }
|
|
|
|
// const table_collection = database.collection(table);
|
|
|
|
// let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
|
|
|
|
// const item_ids_to_remove =
|
|
|
|
// for (let i = 0; i < item_ids_to_update.length; i++) {
|
|
// const item_id_to_update = item_ids_to_update[i];
|
|
// bulk.find({[key_label]: item_id_to_update }).updateOne({
|
|
// "$set": update_obj[item_id_to_update]
|
|
// });
|
|
// }
|
|
|
|
// const output = await bulk.execute();
|
|
// return !!(output['result']['ok']);
|
|
// }
|
|
|
|
|
|
exports.findDuplicatesByKey = async (table, key) => {
|
|
let duplicates = [];
|
|
if (using_local_db) {
|
|
// this can probably be optimized
|
|
const all_records = await exports.getRecords(table);
|
|
const existing_records = {};
|
|
for (let i = 0; i < all_records.length; i++) {
|
|
const record = all_records[i];
|
|
const value = record[key];
|
|
|
|
if (existing_records[value]) {
|
|
duplicates.push(record);
|
|
}
|
|
|
|
existing_records[value] = true;
|
|
}
|
|
return duplicates;
|
|
}
|
|
|
|
const duplicated_values = await database.collection(table).aggregate([
|
|
{"$group" : { "_id": `$${key}`, "count": { "$sum": 1 } } },
|
|
{"$match": {"_id" :{ "$ne" : null } , "count" : {"$gt": 1} } },
|
|
{"$project": {[key] : "$_id", "_id" : 0} }
|
|
]).toArray();
|
|
|
|
for (let i = 0; i < duplicated_values.length; i++) {
|
|
const duplicated_value = duplicated_values[i];
|
|
const duplicated_records = await exports.getRecords(table, duplicated_value, false);
|
|
if (duplicated_records.length > 1) {
|
|
duplicates = duplicates.concat(duplicated_records.slice(1, duplicated_records.length));
|
|
}
|
|
}
|
|
return duplicates;
|
|
}
|
|
|
|
exports.removeAllRecords = async (table = null, filter_obj = null) => {
|
|
// local db override
|
|
const tables_to_remove = table ? [table] : tables_list;
|
|
logger.debug(`Removing all records from: ${tables_to_remove} with filter: ${JSON.stringify(filter_obj)}`)
|
|
if (using_local_db) {
|
|
for (let i = 0; i < tables_to_remove.length; i++) {
|
|
const table_to_remove = tables_to_remove[i];
|
|
if (filter_obj) exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
|
|
else local_db.assign({[table_to_remove]: []}).write();
|
|
logger.debug(`Successfully removed records from ${table_to_remove}`);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
let success = true;
|
|
for (let i = 0; i < tables_to_remove.length; i++) {
|
|
const table_to_remove = tables_to_remove[i];
|
|
|
|
const output = await database.collection(table_to_remove).deleteMany(filter_obj ? filter_obj : {});
|
|
logger.debug(`Successfully removed records from ${table_to_remove}`);
|
|
success &= !!(output['result']['ok']);
|
|
}
|
|
return success;
|
|
}
|
|
|
|
// Stats
|
|
|
|
exports.getDBStats = async () => {
|
|
const stats_by_table = {};
|
|
for (let i = 0; i < tables_list.length; i++) {
|
|
const table = tables_list[i];
|
|
if (table === 'test') continue;
|
|
|
|
stats_by_table[table] = await getDBTableStats(table);
|
|
}
|
|
return {stats_by_table: stats_by_table, using_local_db: using_local_db};
|
|
}
|
|
|
|
const getDBTableStats = async (table) => {
|
|
const table_stats = {};
|
|
// local db override
|
|
if (using_local_db) {
|
|
table_stats['records_count'] = local_db.get(table).value().length;
|
|
} else {
|
|
const stats = await database.collection(table).stats();
|
|
table_stats['records_count'] = stats.count;
|
|
}
|
|
return table_stats;
|
|
}
|
|
|
|
// JSON to DB
|
|
|
|
exports.generateJSONTables = async (db_json, users_json) => {
|
|
// create records
|
|
let files = db_json['files'] || [];
|
|
let playlists = db_json['playlists'] || [];
|
|
let categories = db_json['categories'] || [];
|
|
let subscriptions = db_json['subscriptions'] || [];
|
|
|
|
const users = users_json['users'];
|
|
|
|
for (let i = 0; i < users.length; i++) {
|
|
const user = users[i];
|
|
|
|
if (user['files']) {
|
|
user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] }));
|
|
files = files.concat(user['files']);
|
|
}
|
|
if (user['playlists']) {
|
|
user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] }));
|
|
playlists = playlists.concat(user['playlists']);
|
|
}
|
|
if (user['categories']) {
|
|
user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] }));
|
|
categories = categories.concat(user['categories']);
|
|
}
|
|
|
|
if (user['subscriptions']) {
|
|
user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] }));
|
|
subscriptions = subscriptions.concat(user['subscriptions']);
|
|
}
|
|
}
|
|
|
|
const tables_obj = {};
|
|
|
|
// TODO: use create*Records funcs to strip unnecessary properties
|
|
tables_obj.files = createFilesRecords(files, subscriptions);
|
|
tables_obj.playlists = playlists;
|
|
tables_obj.categories = categories;
|
|
tables_obj.subscriptions = createSubscriptionsRecords(subscriptions);
|
|
tables_obj.users = createUsersRecords(users);
|
|
tables_obj.roles = createRolesRecords(users_json['roles']);
|
|
tables_obj.downloads = createDownloadsRecords(db_json['downloads'])
|
|
|
|
return tables_obj;
|
|
}
|
|
|
|
exports.importJSONToDB = async (db_json, users_json) => {
|
|
await fs.writeFile(`appdata/db.json.${Date.now()/1000}.bak`, JSON.stringify(db_json, null, 2));
|
|
await fs.writeFile(`appdata/users_db.json.${Date.now()/1000}.bak`, JSON.stringify(users_json, null, 2));
|
|
|
|
await exports.removeAllRecords();
|
|
const tables_obj = await exports.generateJSONTables(db_json, users_json);
|
|
|
|
const table_keys = Object.keys(tables_obj);
|
|
|
|
let success = true;
|
|
for (let i = 0; i < table_keys.length; i++) {
|
|
const table_key = table_keys[i];
|
|
if (!tables_obj[table_key] || tables_obj[table_key].length === 0) continue;
|
|
success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key], true);
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
const createFilesRecords = (files, subscriptions) => {
|
|
for (let i = 0; i < subscriptions.length; i++) {
|
|
const subscription = subscriptions[i];
|
|
if (!subscription['videos']) continue;
|
|
subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
|
|
files = files.concat(subscriptions[i]['videos']);
|
|
}
|
|
|
|
return files;
|
|
}
|
|
|
|
const createPlaylistsRecords = async (playlists) => {
|
|
|
|
}
|
|
|
|
const createCategoriesRecords = async (categories) => {
|
|
|
|
}
|
|
|
|
const createSubscriptionsRecords = (subscriptions) => {
|
|
for (let i = 0; i < subscriptions.length; i++) {
|
|
delete subscriptions[i]['videos'];
|
|
}
|
|
|
|
return subscriptions;
|
|
}
|
|
|
|
const createUsersRecords = (users) => {
|
|
users.forEach(user => {
|
|
delete user['files'];
|
|
delete user['playlists'];
|
|
delete user['subscriptions'];
|
|
});
|
|
return users;
|
|
}
|
|
|
|
const createRolesRecords = (roles) => {
|
|
const new_roles = [];
|
|
Object.keys(roles).forEach(role_key => {
|
|
new_roles.push({
|
|
key: role_key,
|
|
...roles[role_key]
|
|
});
|
|
});
|
|
return new_roles;
|
|
}
|
|
|
|
const createDownloadsRecords = (downloads) => {
|
|
const new_downloads = [];
|
|
Object.keys(downloads).forEach(session_key => {
|
|
new_downloads.push({
|
|
key: session_key,
|
|
...downloads[session_key]
|
|
});
|
|
});
|
|
return new_downloads;
|
|
}
|
|
|
|
exports.backupDB = async () => {
|
|
const backup_dir = path.join('appdata', 'db_backup');
|
|
fs.ensureDirSync(backup_dir);
|
|
const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
|
|
const path_to_backups = path.join(backup_dir, backup_file_name);
|
|
|
|
logger.info(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
|
|
|
|
const table_to_records = {};
|
|
for (let i = 0; i < tables_list.length; i++) {
|
|
const table = tables_list[i];
|
|
table_to_records[table] = await exports.getRecords(table);
|
|
}
|
|
|
|
fs.writeJsonSync(path_to_backups, table_to_records);
|
|
|
|
return backup_file_name;
|
|
}
|
|
|
|
exports.restoreDB = async (file_name) => {
|
|
const path_to_backup = path.join('appdata', 'db_backup', file_name);
|
|
|
|
logger.debug('Reading database backup file.');
|
|
const table_to_records = fs.readJSONSync(path_to_backup);
|
|
|
|
if (!table_to_records) {
|
|
logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`);
|
|
return false;
|
|
}
|
|
|
|
logger.debug('Clearing database.');
|
|
await exports.removeAllRecords();
|
|
|
|
logger.debug('Database cleared! Beginning restore.');
|
|
let success = true;
|
|
for (let i = 0; i < tables_list.length; i++) {
|
|
const table = tables_list[i];
|
|
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
|
|
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
|
|
}
|
|
|
|
logger.debug('Restore finished!');
|
|
|
|
return success;
|
|
}
|
|
|
|
exports.transferDB = async (local_to_remote) => {
|
|
const table_to_records = {};
|
|
for (let i = 0; i < tables_list.length; i++) {
|
|
const table = tables_list[i];
|
|
table_to_records[table] = await exports.getRecords(table);
|
|
}
|
|
|
|
logger.info('Backup up DB...');
|
|
await exports.backupDB(); // should backup always
|
|
|
|
using_local_db = !local_to_remote;
|
|
if (local_to_remote) {
|
|
const db_connected = await exports.connectToDB(5, true);
|
|
if (!db_connected) {
|
|
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
|
|
return false;
|
|
}
|
|
}
|
|
success = true;
|
|
|
|
logger.debug('Clearing new database before transfer...');
|
|
|
|
await exports.removeAllRecords();
|
|
|
|
logger.debug('Database cleared! Beginning transfer.');
|
|
|
|
for (let i = 0; i < tables_list.length; i++) {
|
|
const table = tables_list[i];
|
|
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
|
|
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
|
|
}
|
|
|
|
config_api.setConfigItem('ytdl_use_local_db', using_local_db);
|
|
|
|
logger.debug('Transfer finished!');
|
|
|
|
return success;
|
|
}
|
|
|
|
/*
|
|
This function is necessary to emulate mongodb's ability to search for null or missing values.
|
|
A filter of null or undefined for a property will find docs that have that property missing, or have it
|
|
null or undefined. We want that same functionality for the local DB as well
|
|
|
|
error: {$ne: null}
|
|
^ ^
|
|
| |
|
|
filter_prop filter_prop_value
|
|
*/
|
|
exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
|
|
const filter_props = Object.keys(filter_obj);
|
|
const return_val = db_path[operation](record => {
|
|
if (!filter_props) return true;
|
|
let filtered = true;
|
|
for (let i = 0; i < filter_props.length; i++) {
|
|
const filter_prop = filter_props[i];
|
|
const filter_prop_value = filter_obj[filter_prop];
|
|
if (filter_prop_value === undefined || filter_prop_value === null) {
|
|
filtered &= record[filter_prop] === undefined || record[filter_prop] === null;
|
|
} else {
|
|
if (typeof filter_prop_value === 'object') {
|
|
if ('$regex' in filter_prop_value) {
|
|
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
|
|
} else if ('$ne' in filter_prop_value) {
|
|
filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
|
|
} else if ('$lt' in filter_prop_value) {
|
|
filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
|
|
} else if ('$gt' in filter_prop_value) {
|
|
filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
|
|
} else if ('$lte' in filter_prop_value) {
|
|
filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
|
|
} else if ('$gte' in filter_prop_value) {
|
|
filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
|
|
}
|
|
} else {
|
|
// handle case of nested property check
|
|
if (filter_prop.includes('.'))
|
|
filtered &= utils.searchObjectByString(record, filter_prop) === filter_prop_value;
|
|
else
|
|
filtered &= record[filter_prop] === filter_prop_value;
|
|
}
|
|
}
|
|
}
|
|
return filtered;
|
|
});
|
|
return return_val;
|
|
}
|
|
|
|
// should only be used for tests
|
|
exports.setLocalDBMode = (mode) => {
|
|
using_local_db = mode;
|
|
} |