From bb6503e86d16d13f26018f0f0d85a45c82cdff03 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 16 Jul 2021 00:05:08 -0600 Subject: [PATCH] Changed DB structure again Added support for MongoDB Added tests relating to new DB system Category rules are now case insensitive Fixed playlist modification change state --- backend/app.js | 451 +++-------- backend/appdata/default.json | 4 + backend/authentication/auth.js | 138 ++-- backend/categories.js | 17 +- backend/config.js | 4 + backend/consts.js | 10 + backend/db.js | 766 +++++++++++++++--- backend/package-lock.json | 77 ++ backend/package.json | 4 +- backend/subscriptions.js | 164 ++-- backend/test/tests.js | 165 +++- backend/utils.js | 70 ++ .../downloads/downloads.component.html | 16 +- .../downloads/downloads.component.ts | 12 +- .../manage-role/manage-role.component.html | 2 +- .../manage-role/manage-role.component.ts | 2 +- .../modify-users/modify-users.component.html | 2 +- .../modify-users/modify-users.component.ts | 11 +- .../modify-playlist.component.ts | 2 +- src/app/main/main.component.html | 13 +- src/app/main/main.component.ts | 56 +- src/app/player/player.component.html | 4 +- src/app/player/player.component.ts | 9 +- src/app/posts.services.ts | 17 +- src/app/settings/settings.component.html | 37 + src/app/settings/settings.component.scss | 13 +- src/app/settings/settings.component.ts | 59 ++ 27 files changed, 1426 insertions(+), 699 deletions(-) diff --git a/backend/app.js b/backend/app.js index 10db1ac..48f052d 100644 --- a/backend/app.js +++ b/backend/app.js @@ -79,9 +79,9 @@ const logger = winston.createLogger({ }); config_api.initialize(logger); -auth_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); +auth_api.initialize(db_api, logger); +subscriptions_api.initialize(db_api, logger); categories_api.initialize(db, users_db, logger, db_api); // Set some defaults @@ -183,7 +183,7 @@ if (writeConfigMode) { loadConfig(); } -var downloads = {}; +var downloads = []; app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); @@ -210,10 +210,12 @@ async function checkMigrations() { if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); - success = success && await addMetadataPropertyToDB('view_count'); - success = success && await addMetadataPropertyToDB('description'); - success = success && await addMetadataPropertyToDB('height'); - success = success && await addMetadataPropertyToDB('abr'); + success = success && await db_api.addMetadataPropertyToDB('view_count'); + success = success && await db_api.addMetadataPropertyToDB('description'); + success = success && await db_api.addMetadataPropertyToDB('height'); + success = success && await db_api.addMetadataPropertyToDB('abr'); + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } @@ -290,28 +292,6 @@ async function simplifyDBFileStructure() { return true; } -async function addMetadataPropertyToDB(property_key) { - try { - const dirs_to_check = db_api.getFileDirectoriesAndDBs(); - for (const dir_to_check of dirs_to_check) { - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); - for (const file of files) { - if (file[property_key]) { - dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); - } - } - } - - // sets migration to complete - db.set('simplified_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; - } -} - async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku @@ -612,6 +592,9 @@ async function setConfigFromEnv() { async function loadConfig() { loadConfigValues(); + // connect to DB + await db_api.connectToDB(); + // creates archive path if missing await fs.ensureDir(archivePath); @@ -624,7 +607,7 @@ async function loadConfig() { // get subscriptions if (allowSubscriptions) { // set downloading to false - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds watchSubscriptions(); @@ -633,10 +616,10 @@ async function loadConfig() { }, subscriptionsCheckInterval * 1000); } - db_api.importUnregisteredFiles(); + await db_api.importUnregisteredFiles(); // load in previous downloads - downloads = db.get('downloads').value(); + downloads = await db_api.getRecords('downloads'); // start the server here startServer(); @@ -684,7 +667,7 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) { } async function watchSubscriptions() { - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; @@ -903,8 +886,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // adds download to download helper const download_uid = uuid(); const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { + let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session); + if (!session_downloads) { + session_downloads = {session_id: session}; + downloads.push(session_downloads); + } + session_downloads[download_uid] = { uid: download_uid, ui_uid: options.ui_uid, downloading: true, @@ -916,7 +903,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { timestamp_start: Date.now(), filesize: null }; - const download = downloads[session][download_uid]; + const download = session_downloads[download_uid]; updateDownloads(); let download_checker = null; @@ -1027,7 +1014,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } // registers file in DB - const file_obj = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings); + const file_obj = await db_api.registerFileDB2(full_file_path, options.user, category, null, options.cropFileSettings); // TODO: remove the following line if (file_name) file_names.push(file_name); @@ -1070,146 +1057,6 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { }); } -async function downloadFileByURL_normal(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - var file_uid = null; - const is_audio = type === 'audio'; - const ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; } - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'normal'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - const video = youtubedl(url, - // Optional arguments passed to youtube-dl. - downloadConfig, - // Additional options can be given for calling `child_process.execFile()`. - { cwd: __dirname }); - - let video_info = null; - let file_size = 0; - - // Will be called when the download starts. - video.on('info', function(info) { - video_info = info; - file_size = video_info.size; - const json_path = utils.removeFileExtension(video_info._filename) + '.info.json'; - fs.ensureFileSync(json_path); - fs.writeJSONSync(json_path, video_info); - video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) - }); - // Will be called if download was already completed and there is nothing more to download. - video.on('complete', function complete(info) { - 'use strict' - logger.info('file ' + info._filename + ' already downloaded.') - }) - - let download_pos = 0; - video.on('data', function data(chunk) { - download_pos += chunk.length - // `size` should not be 0 here. - if (file_size) { - let percent = (download_pos / file_size * 100).toFixed(2) - download['percent_complete'] = percent; - } - }); - - video.on('end', async function() { - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - download['timestamp_end'] = Date.now(); - download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext]; - download['complete'] = true; - updateDownloads(); - - // audio-only cleanup - if (is_audio) { - // filename fix - video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3'; - - // ID3 tagging - let tags = { - title: video_info['title'], - artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] - } - let success = NodeID3.write(tags, video_info._filename); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); - - const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm'; - const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4'; - // check if audio file is webm - if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']); - else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']); - } - - // registers file in DB - const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name)); - - resolve({ - encodedPath: videopathEncoded, - file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready - uid: file_uid - }); - }); - - video.on('error', function error(err) { - logger.error(err); - - download[error] = err; - updateDownloads(); - - resolve(false); - }); - }); - -} - async function generateArgs(url, type, options) { var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; var globalArgs = config_api.getConfigItem('ytdl_custom_args'); @@ -1250,8 +1097,8 @@ async function generateArgs(url, type, options) { qualityPath = ['-f', customQualityConfiguration]; } else if (selectedHeight && selectedHeight !== '' && !is_audio) { qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; - } else if (maxBitrate && is_audio) { - qualityPath = ['--audio-quality', maxBitrate] + } else if (is_audio) { + qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0'] } if (customOutput) { @@ -1401,23 +1248,6 @@ async function getUrlInfos(urls) { // ffmpeg helper functions -async function convertFileToMp3(input_file, output_file) { - logger.verbose(`Converting ${input_file} to ${output_file}...`); - return new Promise(resolve => { - ffmpeg(input_file).noVideo().toFormat('mp3') - .on('end', () => { - logger.verbose(`Conversion for '${output_file}' complete.`); - fs.unlinkSync(input_file) - resolve(true); - }) - .on('error', (err) => { - logger.error('Failed to convert audio file to the correct format.'); - logger.error(err); - resolve(false); - }).save(output_file); - }); -} - async function cropFile(file_path, start, end, ext) { return new Promise(resolve => { const temp_file_path = `${file_path}.cropped${ext}`; @@ -1456,8 +1286,9 @@ async function writeToBlacklist(type, line) { // download management functions -function updateDownloads() { - db.assign({downloads: downloads}).write(); +async function updateDownloads() { + await db_api.removeAllRecords('downloads'); + if (downloads.length !== 0) await db_api.insertRecordsIntoTable('downloads', downloads); } function checkDownloadPercent(download) { @@ -1489,7 +1320,6 @@ function checkDownloadPercent(download) { } }); download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2); - updateDownloads(); }); } @@ -1715,6 +1545,37 @@ app.post('/api/restartServer', optionalJwt, (req, res) => { res.send({success: true}); }); +app.post('/api/getDBInfo', optionalJwt, async (req, res) => { + const db_info = await db_api.getDBStats(); + res.send({db_info: db_info}); +}); + +app.post('/api/transferDB', optionalJwt, async (req, res) => { + const local_to_remote = req.body.local_to_remote; + let success = null; + let error = ''; + if (local_to_remote === config_api.getConfigItem('ytdl_use_local_db')) { + success = await db_api.transferDB(local_to_remote); + if (!success) error = 'Unknown error'; + else config_api.setConfigItem('ytdl_use_local_db', !local_to_remote); + } else { + success = false; + error = `Failed to transfer DB as it cannot transition into its current status: ${local_to_remote ? 'MongoDB' : 'Local DB'}`; + logger.error(error); + } + + res.send({success: success, error: error}); +}); + +app.post('/api/testConnectionString', optionalJwt, async (req, res) => { + let success = null; + let error = ''; + success = await db_api.connectToDB(5, true); + if (!success) error = 'Connection string failed.'; + + res.send({success: success, error: error}); +}); + app.post('/api/downloadFile', optionalJwt, async function(req, res) { req.setTimeout(0); // remove timeout in case of long videos const url = req.body.url; @@ -1731,15 +1592,7 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) { cropFileSettings: req.body.cropFileSettings } - const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload(); - if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.'); - const is_playlist = url.includes('playlist'); - - let result_obj = null; - if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu')) - result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID); - else - result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID); + let result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID); if (result_obj) { res.send(result_obj); } else { @@ -1752,29 +1605,17 @@ app.post('/api/killAllDownloads', optionalJwt, async function(req, res) { res.send(result_obj); }); -/** - * add thumbnails if present - * @param files - List of files with thumbnailPath property. - */ -async function addThumbnails(files) { - await Promise.all(files.map(async file => { - const thumbnailPath = file['thumbnailPath']; - if (thumbnailPath && (await fs.pathExists(thumbnailPath))) { - file['thumbnailBlob'] = await fs.readFile(thumbnailPath); - } - })); -} - // gets all download mp3s app.get('/api/getMp3s', optionalJwt, async function(req, res) { - var mp3s = db.get('files').value().filter(file => file.isAudio === true); - var playlists = db.get('playlists').value(); + // TODO: simplify + let mp3s = await db_api.getRecords('files', {isAudio: true}); + let playlists = await db_api.getRecords('playlists'); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user audio files/playlists auth_api.passport.authenticate('jwt') - mp3s = auth_api.getUserVideos(req.user.uid, 'audio'); - playlists = auth_api.getUserPlaylists(req.user.uid); + mp3s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: true}); + playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove? } mp3s = JSON.parse(JSON.stringify(mp3s)); @@ -1787,15 +1628,15 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { // gets all download mp4s app.get('/api/getMp4s', optionalJwt, async function(req, res) { - var mp4s = db.get('files').value().filter(file => file.isAudio === false); - var playlists = db.get('playlists').value(); + let mp4s = await db_api.getRecords('files', {isAudio: false}); + let playlists = await db_api.getRecords('playlists'); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user videos/playlists auth_api.passport.authenticate('jwt') - mp4s = auth_api.getUserVideos(req.user.uid, 'video'); - playlists = auth_api.getUserPlaylists(req.user.uid); + mp4s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: false}); + playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove? } mp4s = JSON.parse(JSON.stringify(mp4s)); @@ -1806,20 +1647,14 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) { }); }); -app.post('/api/getFile', optionalJwt, function (req, res) { +app.post('/api/getFile', optionalJwt, async function (req, res) { var uid = req.body.uid; var type = req.body.type; var uuid = req.body.uuid; - var file = null; + var file = await db_api.getRecord('files', {uid: uid}); - if (req.isAuthenticated()) { - file = auth_api.getUserVideo(req.user.uid, uid); - } else if (uuid) { - file = auth_api.getUserVideo(uuid, uid, true); - } else { - file = db.get('files').find({uid: uid}).value(); - } + if (uuid && !file['sharingEnabled']) file = null; // check if chat exists for twitch videos if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json'); @@ -1842,18 +1677,12 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { let playlists = null; const uuid = req.isAuthenticated() ? req.user.uid : null; - let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; + let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (await subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; - // get basic info depending on multi-user mode being enabled - if (uuid) { - files = auth_api.getUserVideos(req.user.uid); - playlists = auth_api.getUserPlaylists(req.user.uid, files); - } else { - files = db.get('files').value(); - playlists = JSON.parse(JSON.stringify(db.get('playlists').value())); - } + files = await db_api.getRecords('files', {user_uid: uuid}); + playlists = await db_api.getRecords('playlists', {user_uid: uuid}); - const categories = categories_api.getCategoriesAsPlaylists(files); + const categories = await categories_api.getCategoriesAsPlaylists(files); if (categories) { playlists = playlists.concat(categories); } @@ -1872,11 +1701,6 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { files = JSON.parse(JSON.stringify(files)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(files); - } - res.send({ files: files, playlists: playlists @@ -1952,7 +1776,7 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { }); // video sharing -app.post('/api/enableSharing', optionalJwt, function(req, res) { +app.post('/api/enableSharing', optionalJwt, async (req, res) => { var uid = req.body.uid; var is_playlist = req.body.is_playlist; let success = false; @@ -1968,15 +1792,9 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist) { - db.get(`files`) - .find({uid: uid}) - .assign({sharingEnabled: true}) - .write(); + await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: true}) } else if (is_playlist) { - db.get(`playlists`) - .find({id: uid}) - .assign({sharingEnabled: true}) - .write(); + await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true}); } else if (false) { // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // time they are requested from the subscription directory. @@ -1995,7 +1813,7 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { }); }); -app.post('/api/disableSharing', optionalJwt, function(req, res) { +app.post('/api/disableSharing', optionalJwt, async function(req, res) { var type = req.body.type; var uid = req.body.uid; var is_playlist = req.body.is_playlist; @@ -2012,15 +1830,9 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files`) - .find({uid: uid}) - .assign({sharingEnabled: false}) - .write(); + await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false}) } else if (is_playlist) { - db.get(`playlists`) - .find({id: uid}) - .assign({sharingEnabled: false}) - .write(); + await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false}); } else if (type === 'subscription') { // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // time they are requested from the subscription directory. @@ -2062,7 +1874,7 @@ app.post('/api/incrementViewCount', optionalJwt, async (req, res) => { // categories app.post('/api/getAllCategories', optionalJwt, async (req, res) => { - const categories = db.get('categories').value(); + const categories = await db_api.getRecords('categories'); res.send({categories: categories}); }); @@ -2075,7 +1887,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { custom_output: '' }; - db.get('categories').push(new_category).write(); + await db_api.insertRecordIntoTable('categories', new_category); res.send({ new_category: new_category, @@ -2086,7 +1898,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { app.post('/api/deleteCategory', optionalJwt, async (req, res) => { const category_uid = req.body.category_uid; - db.get('categories').remove({uid: category_uid}).write(); + await db_api.removeRecord('categories', {uid: category_uid}); res.send({ success: true @@ -2095,13 +1907,14 @@ app.post('/api/deleteCategory', optionalJwt, async (req, res) => { app.post('/api/updateCategory', optionalJwt, async (req, res) => { const category = req.body.category; - db.get('categories').find({uid: category.uid}).assign(category).write(); + await db_api.updateRecord('categories', {uid: category.uid}, category) res.send({success: true}); }); app.post('/api/updateCategories', optionalJwt, async (req, res) => { const categories = req.body.categories; - db.get('categories').assign(categories).write(); + await db_api.removeAllRecords('categories'); + await db_api.insertRecordsIntoTable('categories', categories); res.send({success: true}); }); @@ -2200,9 +2013,9 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { // get sub from db let subscription = null; if (subID) { - subscription = subscriptions_api.getSubscription(subID, user_uid) + subscription = await subscriptions_api.getSubscription(subID, user_uid) } else if (subName) { - subscription = subscriptions_api.getSubscriptionByName(subName, user_uid) + subscription = await subscriptions_api.getSubscriptionByName(subName, user_uid) } if (!subscription) { @@ -2213,7 +2026,8 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { // get sub videos if (subscription.name && !subscription.streamingOnly) { - var parsed_files = subscription.videos; + var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos; + subscription['videos'] = parsed_files; if (!parsed_files) { parsed_files = []; let base_path = null; @@ -2310,7 +2124,7 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { let user_uid = req.isAuthenticated() ? req.user.uid : null; // get subs from api - let subscriptions = subscriptions_api.getSubscriptions(user_uid); + let subscriptions = await subscriptions_api.getSubscriptions(user_uid); res.send({ subscriptions: subscriptions @@ -2343,7 +2157,8 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { for (let i = 0; i < playlist['uids'].length; i++) { const uid = playlist['uids'][i]; const file_obj = await db_api.getVideo(uid, uuid); - file_objs.push(file_obj); + if (file_obj) file_objs.push(file_obj); + // TODO: remove file from playlist if could not be found } } @@ -2364,10 +2179,7 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { if (req.isAuthenticated()) { auth_api.updatePlaylistFiles(req.user.uid, playlistID, uids); } else { - db.get(`playlists`) - .find({id: playlistID}) - .assign({uids: uids}) - .write(); + await db_api.updateRecord('playlists', {id: playlistID}, {uids: uids}) } success = true; @@ -2393,14 +2205,8 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { let success = null; try { - if (req.isAuthenticated()) { - auth_api.removePlaylist(req.user.uid, playlistID); - } else { - // removes playlist from playlists - db.get(`playlists`) - .remove({id: playlistID}) - .write(); - } + // removes playlist from playlists + await db_api.removeRecord('playlists', {id: playlistID}) success = true; } catch(e) { @@ -2611,20 +2417,14 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { }); app.post('/api/download', async (req, res) => { - var session_id = req.body.session_id; - var download_id = req.body.download_id; + const session_id = req.body.session_id; + const download_id = req.body.download_id; + const session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session_id); let found_download = null; // find download - if (downloads[session_id] && Object.keys(downloads[session_id])) { - let session_downloads = Object.values(downloads[session_id]); - for (let i = 0; i < session_downloads.length; i++) { - let session_download = session_downloads[i]; - if (session_download && session_download['ui_uid'] === download_id) { - found_download = session_download; - break; - } - } + if (session_downloads && Object.keys(session_downloads)) { + found_download = Object.values(session_downloads).find(session_download => session_download['ui_uid'] === download_id); } if (found_download) { @@ -2642,26 +2442,22 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { var download_id = req.body.download_id; if (delete_all) { // delete all downloads - downloads = {}; + downloads = []; success = true; } else if (download_id) { // delete just 1 download - if (downloads[session_id][download_id]) { - delete downloads[session_id][download_id]; + const session_downloads = downloads.find(session => session['session_id'] === session_id); + if (session_downloads && session_downloads[download_id]) { + delete session_downloads[download_id]; success = true; - } else if (!downloads[session_id]) { + } else if (!session_downloads) { logger.error(`Session ${session_id} has no downloads.`) - } else if (!downloads[session_id][download_id]) { + } else if (!session_downloads[download_id]) { logger.error(`Download '${download_id}' for session '${session_id}' could not be found`); } } else if (session_id) { // delete a session's downloads - if (downloads[session_id]) { - delete downloads[session_id]; - success = true; - } else { - logger.error(`Session ${session_id} has no downloads.`) - } + downloads = downloads.filter(session => session['session_id'] !== session_id); } updateDownloads(); res.send({success: success, downloads: downloads}); @@ -2746,29 +2542,28 @@ app.post('/api/auth/changePassword', optionalJwt, async (req, res) => { res.send({success: success}); }); app.post('/api/auth/adminExists', async (req, res) => { - let exists = auth_api.adminExists(); + let exists = await auth_api.adminExists(); res.send({exists: exists}); }); // user management app.post('/api/getUsers', optionalJwt, async (req, res) => { - let users = users_db.get('users').value(); + let users = await db_api.getRecords('users'); res.send({users: users}); }); app.post('/api/getRoles', optionalJwt, async (req, res) => { - let roles = users_db.get('roles').value(); + let roles = await db_api.getRecords('roles'); res.send({roles: roles}); }); app.post('/api/updateUser', optionalJwt, async (req, res) => { let change_obj = req.body.change_object; try { - const user_db_obj = users_db.get('users').find({uid: change_obj.uid}); if (change_obj.name) { - user_db_obj.assign({name: change_obj.name}).write(); + await db_api.updateRecord('users', {uid: change_obj.uid}, {name: change_obj.name}); } if (change_obj.role) { - user_db_obj.assign({role: change_obj.role}).write(); + await db_api.updateRecord('users', {uid: change_obj.uid}, {role: change_obj.role}); } res.send({success: true}); } catch (err) { @@ -2780,13 +2575,17 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => { app.post('/api/deleteUser', optionalJwt, async (req, res) => { let uid = req.body.uid; try { + let success = false; let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const user_folder = path.join(__dirname, usersFileFolder, uid); - const user_db_obj = users_db.get('users').find({uid: uid}); - if (user_db_obj.value()) { + const user_db_obj = await db_api.getRecord('users', {uid: uid}); + if (user_db_obj) { // user exists, let's delete await fs.remove(user_folder); - users_db.get('users').remove({uid: uid}).write(); + await db_api.removeRecord('users', {uid: uid}); + success = true; + } else { + logger.error(`Could not find user with uid ${uid}`); } res.send({success: true}); } catch (err) { @@ -2805,7 +2604,7 @@ app.post('/api/changeUserPermissions', optionalJwt, async (req, res) => { return; } - const success = auth_api.changeUserPermissions(user_uid, permission, new_value); + const success = await auth_api.changeUserPermissions(user_uid, permission, new_value); res.send({success: success}); }); @@ -2820,7 +2619,7 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => { return; } - const success = auth_api.changeRolePermissions(role, permission, new_value); + const success = await auth_api.changeRolePermissions(role, permission, new_value); res.send({success: success}); }); diff --git a/backend/appdata/default.json b/backend/appdata/default.json index 5bd5c54..5c2ba9c 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -54,6 +54,10 @@ "searchFilter": "(uid={{username}})" } }, + "Database": { + "use_local_db": false, + "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" + }, "Advanced": { "default_downloader": "youtube-dl", "use_default_downloading_agent": true, diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 6e83ec3..7992aac 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -13,16 +13,15 @@ var JwtStrategy = require('passport-jwt').Strategy, // other required vars let logger = null; -let db = null; -let users_db = null; +let db_api = null; let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function(input_db, input_users_db, input_logger) { +exports.initialize = function(db_api, input_logger) { setLogger(input_logger) - setDB(input_db, input_users_db); + setDB(db_api); /************************* * Authentication module @@ -32,21 +31,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) { JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration'); SERVER_SECRET = null; - if (users_db.get('jwt_secret').value()) { - SERVER_SECRET = users_db.get('jwt_secret').value(); + if (db_api.users_db.get('jwt_secret').value()) { + SERVER_SECRET = db_api.users_db.get('jwt_secret').value(); } else { SERVER_SECRET = uuid(); - users_db.set('jwt_secret', SERVER_SECRET).write(); + db_api.users_db.set('jwt_secret', SERVER_SECRET).write(); } opts = {} opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt'); opts.secretOrKey = SERVER_SECRET; - /*opts.issuer = 'example.com'; - opts.audience = 'example.com';*/ - exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) { - const user = users_db.get('users').find({uid: jwt_payload.user}).value(); + exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) { + const user = await db_api.getRecord('users', {uid: jwt_payload.user}); if (user) { return done(null, user); } else { @@ -60,9 +57,8 @@ function setLogger(input_logger) { logger = input_logger; } -function setDB(input_db, input_users_db) { - db = input_db; - users_db = input_users_db; +function setDB(input_db_api) { + db_api = input_db_api; } exports.passport = require('passport'); @@ -78,7 +74,7 @@ exports.passport.deserializeUser(function(user, done) { /*************************************** * Register user with hashed password **************************************/ -exports.registerUser = function(req, res) { +exports.registerUser = async function(req, res) { var userid = req.body.userid; var username = req.body.username; var plaintextPassword = req.body.password; @@ -96,20 +92,20 @@ exports.registerUser = function(req, res) { } bcrypt.hash(plaintextPassword, saltRounds) - .then(function(hash) { + .then(async function(hash) { let new_user = generateUserObject(userid, username, hash); // check if user exists - if (users_db.get('users').find({uid: userid}).value()) { + if (await db_api.getRecord('users', {uid: userid})) { // user id is taken! logger.error('Registration failed: UID is already taken!'); res.status(409).send('UID is already taken!'); - } else if (users_db.get('users').find({name: username}).value()) { + } else if (await db_api.getRecord('users', {name: username})) { // user name is taken! logger.error('Registration failed: User name is already taken!'); res.status(409).send('User name is already taken!'); } else { // add to db - users_db.get('users').push(new_user).write(); + await db_api.insertRecordIntoTable('users', new_user); logger.verbose(`New user created: ${new_user.name}`); res.send({ user: new_user @@ -143,7 +139,7 @@ exports.registerUser = function(req, res) { exports.login = async (username, password) => { - const user = users_db.get('users').find({name: username}).value(); + const user = await db_api.getRecord('users', {name: username}); if (!user) { logger.error(`User ${username} not found`); false } if (user.auth_method && user.auth_method !== 'internal') { return false } return await bcrypt.compare(password, user.passhash) ? user : false; @@ -164,17 +160,17 @@ var getLDAPConfiguration = function(req, callback) { }; exports.passport.use(new LdapStrategy(getLDAPConfiguration, - function(user, done) { + async function(user, done) { // check if ldap auth is enabled const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap'; if (!ldap_enabled) return done(null, false); const user_uid = user.uid; - let db_user = users_db.get('users').find({uid: user_uid}).value(); + let db_user = await db_api.getRecord('users', {uid: user_uid}); if (!db_user) { // generate DB user let new_user = generateUserObject(user_uid, user_uid, null, 'ldap'); - users_db.get('users').push(new_user).write(); + await db_api.insertRecordIntoTable('users', new_user); db_user = new_user; logger.verbose(`Generated new user ${user_uid} using LDAP`); } @@ -198,11 +194,11 @@ exports.generateJWT = function(req, res, next) { next(); } -exports.returnAuthResponse = function(req, res) { +exports.returnAuthResponse = async function(req, res) { res.status(200).json({ user: req.user, token: req.token, - permissions: exports.userPermissions(req.user.uid), + permissions: await exports.userPermissions(req.user.uid), available_permissions: consts['AVAILABLE_PERMISSIONS'] }); } @@ -215,7 +211,7 @@ exports.returnAuthResponse = function(req, res) { * It also passes the user object to the next * middleware through res.locals **************************************/ -exports.ensureAuthenticatedElseError = function(req, res, next) { +exports.ensureAuthenticatedElseError = (req, res, next) => { var token = getToken(req.query); if( token ) { try { @@ -233,10 +229,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) { } // change password -exports.changeUserPassword = async function(user_uid, new_pass) { +exports.changeUserPassword = async (user_uid, new_pass) => { try { const hash = await bcrypt.hash(new_pass, saltRounds); - users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write(); + await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash}); return true; } catch (err) { return false; @@ -244,16 +240,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) { } // change user permissions -exports.changeUserPermissions = function(user_uid, permission, new_value) { +exports.changeUserPermissions = async (user_uid, permission, new_value) => { try { - const user_db_obj = users_db.get('users').find({uid: user_uid}); - user_db_obj.get('permissions').pull(permission).write(); - user_db_obj.get('permission_overrides').pull(permission).write(); + await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission); + await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); if (new_value === 'yes') { - user_db_obj.get('permissions').push(permission).write(); - user_db_obj.get('permission_overrides').push(permission).write(); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); } else if (new_value === 'no') { - user_db_obj.get('permission_overrides').push(permission).write(); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); } return true; } catch (err) { @@ -263,12 +258,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) { } // change role permissions -exports.changeRolePermissions = function(role, permission, new_value) { +exports.changeRolePermissions = async (role, permission, new_value) => { try { - const role_db_obj = users_db.get('roles').get(role); - role_db_obj.get('permissions').pull(permission).write(); + await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission); if (new_value === 'yes') { - role_db_obj.get('permissions').push(permission).write(); + await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission); } return true; } catch (err) { @@ -277,19 +271,19 @@ exports.changeRolePermissions = function(role, permission, new_value) { } } -exports.adminExists = function() { - return !!users_db.get('users').find({uid: 'admin'}).value(); +exports.adminExists = async function() { + return !!(await db_api.getRecord('users', {uid: 'admin'})); } // video stuff -exports.getUserVideos = function(user_uid, type) { - const user = users_db.get('users').find({uid: user_uid}).value(); - return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files']; +exports.getUserVideos = async function(user_uid, type) { + const files = await db_api.getRecords('files', {user_uid: user_uid}); + return type ? files.filter(file => file.isAudio === (type === 'audio')) : files; } -exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { - let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); +exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) { + let file = await db_api.getRecord('files', {file_uid: file_uid}); // prevent unauthorized users from accessing the file info if (file && !file['sharingEnabled'] && requireSharing) file = null; @@ -302,19 +296,17 @@ exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) { return true; } -exports.removePlaylist = function(user_uid, playlistID) { - users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write(); +exports.removePlaylist = async function(user_uid, playlistID) { + await db_api.removeRecord('playlist', {playlistID: playlistID}); return true; } -exports.getUserPlaylists = function(user_uid, user_files = null) { - const user = users_db.get('users').find({uid: user_uid}).value(); - const playlists = JSON.parse(JSON.stringify(user['playlists'])); - return playlists; +exports.getUserPlaylists = async function(user_uid, user_files = null) { + return await db_api.getRecords('playlists', {user_uid: user_uid}); } -exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { - let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value(); +exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) { + let playlist = await db_api.getRecord('playlists', {id: playlistID}); // prevent unauthorized users from accessing the file info if (requireSharing && !playlist['sharingEnabled']) playlist = null; @@ -322,40 +314,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) return playlist; } -exports.registerUserFile = function(user_uid, file_object) { - users_db.get('users').find({uid: user_uid}).get(`files`) - .remove({ - path: file_object['path'] - }).write(); - - users_db.get('users').find({uid: user_uid}).get(`files`) - .push(file_object) - .write(); -} - -exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) { +exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) { let success = false; - const user_db_obj = users_db.get('users').find({uid: user_uid}); - if (user_db_obj.value()) { - const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid}); - if (file_db_obj.value()) { - success = true; - file_db_obj.assign({sharingEnabled: enabled}).write(); - } - } - + is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled}); + success = true; return success; } -exports.userHasPermission = function(user_uid, permission) { - const user_obj = users_db.get('users').find({uid: user_uid}).value(); +exports.userHasPermission = async function(user_uid, permission) { + + const user_obj = await db_api.getRecord('users', ({uid: user_uid})); const role = user_obj['role']; if (!role) { // role doesn't exist logger.error('Invalid role ' + role); return false; } - const role_permissions = (users_db.get('roles').value())['permissions']; + const role_permissions = (await db_api.getRecords('roles'))['permissions']; const user_has_explicit_permission = user_obj['permissions'].includes(permission); const permission_in_overrides = user_obj['permission_overrides'].includes(permission); @@ -378,16 +353,17 @@ exports.userHasPermission = function(user_uid, permission) { } } -exports.userPermissions = function(user_uid) { +exports.userPermissions = async function(user_uid) { let user_permissions = []; - const user_obj = users_db.get('users').find({uid: user_uid}).value(); + const user_obj = await db_api.getRecord('users', ({uid: user_uid})); const role = user_obj['role']; if (!role) { // role doesn't exist logger.error('Invalid role ' + role); return null; } - const role_permissions = users_db.get('roles').get(role).get('permissions').value() + const role_obj = await db_api.getRecord('roles', {key: role}); + const role_permissions = role_obj['permissions']; for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) { let permission = consts['AVAILABLE_PERMISSIONS'][i]; diff --git a/backend/categories.js b/backend/categories.js index ce56d5c..fb33a88 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -39,10 +39,9 @@ async function categorize(file_jsons) { if (!Array.isArray(file_jsons)) file_jsons = [file_jsons]; let selected_category = null; - const categories = getCategories(); + const categories = await getCategories(); if (!categories) { - logger.warn('Categories could not be found. Initializing categories...'); - db.assign({categories: []}).write(); + logger.warn('Categories could not be found.'); return null; } @@ -64,14 +63,14 @@ async function categorize(file_jsons) { return selected_category; } -function getCategories() { - const categories = db.get('categories').value(); +async function getCategories() { + const categories = await db_api.getRecords('categories'); return categories ? categories : null; } -function getCategoriesAsPlaylists(files = null) { +async function getCategoriesAsPlaylists(files = null) { const categories_as_playlists = []; - const available_categories = getCategories(); + const available_categories = await getCategories(); if (available_categories && files) { for (category of available_categories) { const files_that_match = utils.addUIDsToCategory(category, files); @@ -97,10 +96,10 @@ function applyCategoryRules(file_json, rules, category_name) { switch (rule['comparator']) { case 'includes': - rule_applies = file_json[rule['property']].includes(rule['value']); + rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()); break; case 'not_includes': - rule_applies = !(file_json[rule['property']].includes(rule['value'])); + rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase())); break; case 'equals': rule_applies = file_json[rule['property']] === rule['value']; diff --git a/backend/config.js b/backend/config.js index cb3e8b3..4a61999 100644 --- a/backend/config.js +++ b/backend/config.js @@ -231,6 +231,10 @@ DEFAULT_CONFIG = { "searchFilter": "(uid={{username}})" } }, + "Database": { + "use_local_db": false, + "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" + }, "Advanced": { "default_downloader": "youtube-dl", "use_default_downloading_agent": true, diff --git a/backend/consts.js b/backend/consts.js index fc29fcf..91cfc75 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -153,6 +153,16 @@ let CONFIG_ITEMS = { 'path': 'YoutubeDLMaterial.Users.ldap_config' }, + // Database + 'ytdl_use_local_db': { + 'key': 'ytdl_use_local_db', + 'path': 'YoutubeDLMaterial.Database.use_local_db' + }, + 'ytdl_mongodb_connection_string': { + 'key': 'ytdl_mongodb_connection_string', + 'path': 'YoutubeDLMaterial.Database.mongodb_connection_string' + }, + // Advanced 'ytdl_default_downloader': { 'key': 'ytdl_default_downloader', diff --git a/backend/db.js b/backend/db.js index 719161c..7bb5258 100644 --- a/backend/db.js +++ b/backend/db.js @@ -3,22 +3,102 @@ var path = require('path') var utils = require('./utils') const { uuid } = require('uuidv4'); const config_api = require('./config'); +const { MongoClient } = require("mongodb"); + +const low = require('lowdb') +const FileSync = require('lowdb/adapters/FileSync'); +const local_adapter = new FileSync('./appdata/local_db.json'); +const local_db = low(local_adapter); var logger = null; var db = null; var users_db = null; -function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db } -function setLogger(input_logger) { logger = input_logger; } +var database = null; + +const tables = ['files', 'playlists', 'categories', 'subscriptions', 'downloads', 'users', 'roles', 'test']; + +const local_db_defaults = {} +tables.forEach(table => {local_db_defaults[table] = []}); +local_db.defaults(local_db_defaults).write(); + +let using_local_db = config_api.getConfigItem('ytdl_use_local_db'); + +function setDB(input_db, input_users_db) { + db = input_db; users_db = input_users_db; + exports.db = input_db; + exports.users_db = input_users_db +} + +function setLogger(input_logger) { + logger = input_logger; +} exports.initialize = (input_db, input_users_db, input_logger) => { setDB(input_db, input_users_db); setLogger(input_logger); } -exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null) => { +exports.connectToDB = async (retries = 5, no_fallback = false) => { + if (using_local_db) return; + const success = await exports._connectToDB(); + if (success) return true; + + logger.warn(`MongoDB connection failed! Retrying ${retries} times...`); + const retry_delay_ms = 2000; + for (let i = 0; i < retries; i++) { + const retry_succeeded = await exports._connectToDB(); + if (retry_succeeded) { + logger.info(`Successfully connected to DB after ${i+1} attempt(s)`); + return true; + } + + if (i !== retries - 1) { + logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`); + await utils.wait(retry_delay_ms); + } else { + logger.warn(`Retry ${i+1} failed.`); + } + } + if (no_fallback) { + logger.error('Failed to connect to MongoDB. Verify your connection string is valid.'); + return; + } + using_local_db = true; + config_api.setConfigItem('ytdl_use_local_db', true); + logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.'); + return true; +} + +exports._connectToDB = async () => { + const uri = config_api.getConfigItem('ytdl_mongodb_connection_string'); // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb"; + const client = new MongoClient(uri, { + useNewUrlParser: true, + useUnifiedTopology: true, + }); + + try { + await client.connect(); + database = client.db('ytdl_material'); + const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name); + + const missing_tables = tables.filter(table => !(existing_collections.includes(table))); + missing_tables.forEach(async table => { + await database.createCollection(table); + }) + return true; + } catch(err) { + logger.error(err); + return false; + } finally { + // Ensures that the client will close when you finish/error + // await client.close(); + } +} + +exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => { let db_path = null; const file_id = utils.removeFileExtension(file_path); - const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); + if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); if (!file_object) { logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); return false; @@ -37,23 +117,9 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; } - if (!sub) { - if (multiUserMode) { - const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get(`files`); - } else { - db_path = db.get(`files`); - } - } else { - if (multiUserMode) { - const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).get('videos'); - } else { - db_path = db.get('subscriptions').find({id: sub.id}).get('videos'); - } - } + if (multiUserMode) file_object['user_uid'] = multiUserMode.user; - const file_obj = registerFileDBManual(db_path, file_object); + const file_obj = await registerFileDBManual(file_object); // remove metadata JSON if needed if (!config_api.getConfigItem('ytdl_include_metadata')) { @@ -63,18 +129,48 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus return file_obj; } -function registerFileDBManual(db_path, file_object) { +exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { + if (!file_object) file_object = generateFileObject2(file_path, type); + if (!file_object) { + logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); + return false; + } + + utils.fixVideoMetadataPerms2(file_path, type); + + // add thumbnail path + file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type); + + // if category exists, only include essential info + if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; + + // modify duration + if (cropFileSettings) { + file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; + } + + if (user_uid) file_object['user_uid'] = user_uid; + if (sub_id) file_object['sub_id'] = sub_id; + + const file_obj = await registerFileDBManual(file_object); + + // remove metadata JSON if needed + if (!config_api.getConfigItem('ytdl_include_metadata')) { + utils.deleteJSONFile2(file_path, type) + } + + return file_obj; +} + +async function registerFileDBManual(file_object) { // add additional info file_object['uid'] = uuid(); file_object['registered'] = Date.now(); path_object = path.parse(file_object['path']); file_object['path'] = path.format(path_object); - // remove duplicate(s) - db_path.remove({path: file_object['path']}).write(); + exports.insertRecordIntoTable('files', file_object, {path: file_object['path']}) - // add new file to db - db_path.push(file_object).write(); return file_object; } @@ -107,11 +203,38 @@ function generateFileObject(id, type, customPath = null, sub = null) { return file_obj; } +function generateFileObject2(file_path, type) { + var jsonobj = utils.getJSON(file_path, type); + if (!jsonobj) { + return null; + } + const ext = (type === 'audio') ? '.mp3' : '.mp4' + const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type); + // console. + var stats = fs.statSync(true_file_path); + + const file_id = utils.removeFileExtension(path.basename(file_path)); + var title = jsonobj.title; + var url = jsonobj.webpage_url; + var uploader = jsonobj.uploader; + var upload_date = jsonobj.upload_date; + upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A'; + + var size = stats.size; + + var thumbnail = jsonobj.thumbnail; + var duration = jsonobj.duration; + var isaudio = type === 'audio'; + var description = jsonobj.description; + var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); + return file_obj; +} + function getAppendedBasePathSub(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } -exports.getFileDirectoriesAndDBs = () => { +exports.getFileDirectoriesAndDBs = async () => { let dirs_to_check = []; let subscriptions_to_check = []; const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode @@ -119,48 +242,45 @@ exports.getFileDirectoriesAndDBs = () => { const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions'); if (multi_user_mode) { - let users = users_db.get('users').value(); + const users = await exports.getRecords('users'); for (let i = 0; i < users.length; i++) { const user = users[i]; - if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']); - // add user's audio dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'audio'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files'), + user_uid: user.uid, type: 'audio' }); // add user's video dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'video'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'video' }); } } else { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - const subscriptions = db.get('subscriptions').value(); - - if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions); // add audio dir to check list dirs_to_check.push({ basePath: audioFolderPath, - dbPath: db.get('files'), type: 'audio' }); // add video dir to check list dirs_to_check.push({ basePath: videoFolderPath, - dbPath: db.get('files'), type: 'video' }); } + if (subscriptions_enabled) { + const subscriptions = await exports.getRecords('subscriptions'); + subscriptions_to_check = subscriptions_to_check.concat(subscriptions); + } + // add subscriptions to check list for (let i = 0; i < subscriptions_to_check.length; i++) { let subscription_to_check = subscriptions_to_check[i]; @@ -169,11 +289,11 @@ exports.getFileDirectoriesAndDBs = () => { continue; } dirs_to_check.push({ - basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) + basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name), - dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos') - : db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'), - type: subscription_to_check.type + user_uid: subscription_to_check.user_uid, + type: subscription_to_check.type, + sub_id: subscription_to_check['id'] }); } @@ -181,22 +301,25 @@ exports.getFileDirectoriesAndDBs = () => { } exports.importUnregisteredFiles = async () => { - const dirs_to_check = exports.getFileDirectoriesAndDBs(); + const dirs_to_check = await exports.getFileDirectoriesAndDBs(); // run through check list and check each file to see if it's missing from the db - for (const dir_to_check of dirs_to_check) { + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; // recursively get all files in dir's path const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); - files.forEach(file => { + for (let j = 0; j < files.length; j++) { + const file = files[j]; + // check if file exists in db, if not add it - const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value()) + const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id})) if (!file_is_registered) { // add additional info - registerFileDBManual(dir_to_check.dbPath, file); + await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); logger.verbose(`Added discovered file to the database: ${file.id}`); } - }); + } } } @@ -204,26 +327,45 @@ exports.importUnregisteredFiles = async () => { exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => { const preimported_file_paths = []; - let dbPath = null; - if (sub.user_uid) - dbPath = users_db.get('users').find({uid: sub.user_uid}).get('subscriptions').find({id: sub.id}).get('videos'); - else - dbPath = db.get('subscriptions').find({id: sub.id}).get('videos'); - const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type); - files.forEach(file => { + for (let i = 0; i < files.length; i++) { + const file = files[i]; // check if file exists in db, if not add it - const file_is_registered = !!(dbPath.find({id: file.id}).value()) + const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id}); if (!file_is_registered) { // add additional info - registerFileDBManual(dbPath, file); + await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file); preimported_file_paths.push(file['path']); logger.verbose(`Preemptively added subscription file to the database: ${file.id}`); } - }); + } return preimported_file_paths; } +exports.addMetadataPropertyToDB = async (property_key) => { + try { + const dirs_to_check = await exports.getFileDirectoriesAndDBs(); + const update_obj = {}; + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; + + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); + for (let j = 0; j < files.length; j++) { + const file = files[j]; + if (file[property_key]) { + update_obj[file.uid] = {[property_key]: file[property_key]}; + } + } + } + + return await exports.bulkUpdateRecords('files', 'uid', update_obj); + } catch(err) { + logger.error(err); + return false; + } +} + exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => { let new_playlist = { name: playlist_name, @@ -236,28 +378,19 @@ exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_u const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid); new_playlist.duration = duration; - - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write(); - } else { - db.get(`playlists`) - .push(new_playlist) - .write(); - } + + new_playlist.user_uid = user_uid ? user_uid : undefined; + + await exports.insertRecordIntoTable('playlists', new_playlist); return new_playlist; } exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => { - let playlist = null - if (user_uid) { - playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlist_id}).value(); - } else { - playlist = db.get(`playlists`).find({id: playlist_id}).value(); - } + let playlist = await exports.getRecord('playlists', {id: playlist_id}); if (!playlist) { - playlist = db.get('categories').find({uid: playlist_id}).value(); + playlist = await exports.getRecord('categories', {uid: playlist_id}); if (playlist) { // category found const files = await exports.getFiles(user_uid); @@ -271,7 +404,7 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`); for (let i = 0; i < playlist['fileNames'].length; i++) { const fileName = playlist['fileNames'][i]; - const uid = exports.getVideoUIDByID(fileName, user_uid); + const uid = await exports.getVideoUIDByID(fileName, user_uid); if (uid) playlist['uids'].push(uid); else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); } @@ -290,14 +423,21 @@ exports.updatePlaylist = async (playlist, user_uid = null) => { const duration = await exports.calculatePlaylistDuration(playlist, user_uid); playlist.duration = duration; - let db_loc = null; - if (user_uid) { - db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}); - } else { - db_loc = db.get(`playlists`).find({id: playlistID}); + return await exports.updateRecord('playlists', {id: playlistID}, playlist); +} + +exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => { + let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj); + + if (!success) { + success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj); } - db_loc.assign(playlist).write(); - return true; + + if (!success) { + logger.error(`Could not find playlist or category with ID ${playlist_id}`); + } + + return success; } exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => { @@ -381,8 +521,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => { if (jsonExists) await fs.unlink(jsonPath); if (thumbnailExists) await fs.unlink(thumbnailPath); - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - base_db_path.get('files').remove({uid: uid}).write(); + await exports.removeRecord('files', {uid: uid}); if (fileExists) { await fs.unlink(file_obj.path); @@ -398,29 +537,462 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => { } // Video ID is basically just the file name without the base path and file extension - this method helps us get away from that -exports.getVideoUIDByID = (file_id, uuid = null) => { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - const file_obj = base_db_path.get('files').find({id: file_id}).value(); +exports.getVideoUIDByID = async (file_id, uuid = null) => { + const file_obj = await exports.getRecord('files', {id: file_id}); return file_obj ? file_obj['uid'] : null; } exports.getVideo = async (file_uid, uuid = null, sub_id = null) => { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); - return sub_db_path.find({uid: file_uid}).value(); + return await exports.getRecord('files', {uid: file_uid}); } exports.getFiles = async (uuid = null) => { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - return base_db_path.get('files').value(); + return await exports.getRecords('files', {user_uid: uuid}); } -exports.setVideoProperty = async (file_uid, assignment_obj, uuid, sub_id) => { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); - const file_db_path = sub_db_path.find({uid: file_uid}); - if (!(file_db_path.value())) { - logger.error(`Failed to find file with uid ${file_uid}`); - } - sub_db_path.find({uid: file_uid}).assign(assignment_obj).write(); +exports.setVideoProperty = async (file_uid, assignment_obj) => { + // TODO: check if video exists, throw error if not + await exports.updateRecord('files', {uid: file_uid}, assignment_obj); } + +// DB to JSON + +exports.exportDBToJSON = async (tables) => { + const users_db_json = await createUsersJSONs(tables.files, tables.playlists, tables.subscriptions, tables.categories, tables.users); + const db_json = await createNonUserJSON(tables.files, tables.playlists, tables.subscriptions, tables.categories); + + return {users_db_json: users_db_json, db_json: db_json}; +} + +const createUsersJSONs = async (files, playlists, subscriptions, categories, users) => { + // we need to already have a list of user objects to gather the records into + for (let user of users) { + const files_of_user = files.filter(file => file.user_uid === user.uid && !file.sub_id); + const playlists_of_user = playlists.filter(playlist => playlist.user_uid === user.uid); + const subs_of_user = subscriptions.filter(sub => sub.user_uid === user.uid); + const categories_of_user = categories ? categories.filter(category => category && category.user_uid === user.uid) : []; + user['files'] = files_of_user; + user['playlists'] = playlists_of_user; + user['subscriptions'] = subs_of_user; + user['categories'] = categories_of_user; + + for (let subscription of subscriptions) { + subscription['videos'] = files.filter(file => file.user_uid === user.uid && file.sub_id === sub.id); + } + } +} + +const createNonUserJSON = async (files, playlists, subscriptions, categories) => { + const non_user_json = { + files: files.filter(file => !file.user_uid && !file.sub_id), + playlists: playlists.filter(playlist => !playlist.user_uid), + subscriptions: subscriptions.filter(sub => !sub.user_uid), + categories: categories ? categories.filter(category => category && !category.user_uid) : [] + } + + for (let subscription of non_user_json['subscriptions']) { + subscription['videos'] = files.filter(file => !file.user_uid && file.sub_id === subscription.id); + } + + return non_user_json; +} + +// Basic DB functions + +// Create + +exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => { + // local db override + if (using_local_db) { + if (replaceFilter) local_db.get(table).remove(replaceFilter).write(); + local_db.get(table).push(doc).write(); + return true; + } + + if (replaceFilter) await database.collection(table).deleteMany(replaceFilter); + + const output = await database.collection(table).insertOne(doc); + logger.debug(`Inserted doc into ${table}`); + return !!(output['result']['ok']); +} + +exports.insertRecordsIntoTable = async (table, docs) => { + // local db override + if (using_local_db) { + local_db.get(table).push(...docs).write(); + return true; + } + + const output = await database.collection(table).insertMany(docs); + logger.debug(`Inserted ${output.insertedCount} docs into ${table}`); + return !!(output['result']['ok']); +} + +exports.bulkInsertRecordsIntoTable = async (table, docs) => { + // local db override + if (using_local_db) { + return await exports.insertRecordsIntoTable(table, docs); + } + + // not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed + const table_collection = database.collection(table); + + let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch + + for (let i = 0; i < docs.length; i++) { + bulk.insert(docs[i]); + } + + const output = await bulk.execute(); + return !!(output['result']['ok']); + +} + +// Read + +exports.getRecord = async (table, filter_obj) => { + // local db override + if (using_local_db) { + return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value(); + } + + return await database.collection(table).findOne(filter_obj); +} + +exports.getRecords = async (table, filter_obj = null) => { + // local db override + if (using_local_db) { + return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value(); + } + + return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray(); +} + +// Update + +exports.updateRecord = async (table, filter_obj, update_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write(); + return true; + } + + // sometimes _id will be in the update obj, this breaks mongodb + if (update_obj['_id']) delete update_obj['_id']; + const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj}); + return !!(output['result']['ok']); +} + +exports.updateRecords = async (table, filter_obj, update_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write(); + return true; + } + + const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj}); + return !!(output['result']['ok']); +} + +exports.bulkUpdateRecords = async (table, key_label, update_obj) => { + // local db override + if (using_local_db) { + local_db.get(table).each((record) => { + const item_id_to_update = record[key_label]; + if (!update_obj[item_id_to_update]) return; + + const props_to_update = Object.keys(update_obj[item_id_to_update]); + for (let i = 0; i < props_to_update.length; i++) { + const prop_to_update = props_to_update[i]; + const prop_value = update_obj[item_id_to_update][prop_to_update]; + record[prop_to_update] = prop_value; + } + }).write(); + return true; + } + + const table_collection = database.collection(table); + + let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch + + const item_ids_to_update = Object.keys(update_obj); + + for (let i = 0; i < item_ids_to_update.length; i++) { + const item_id_to_update = item_ids_to_update[i]; + bulk.find({[key_label]: item_id_to_update }).updateOne({ + "$set": update_obj[item_id_to_update] + }); + } + + const output = await bulk.execute(); + return !!(output['result']['ok']); +} + +exports.pushToRecordsArray = async (table, filter_obj, key, value) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write(); + return true; + } + + const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}}); + return !!(output['result']['ok']); +} + +exports.pullFromRecordsArray = async (table, filter_obj, key, value) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write(); + return true; + } + + const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}}); + return !!(output['result']['ok']); +} + +// Delete + +exports.removeRecord = async (table, filter_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write(); + return true; + } + + const output = await database.collection(table).deleteOne(filter_obj); + return !!(output['result']['ok']); +} + +exports.removeAllRecords = async (table = null) => { + // local db override + if (using_local_db) { + const tables_to_remove = table ? [table] : tables; + logger.debug(`Removing all records from: ${tables_to_remove}`) + for (let i = 0; i < tables_to_remove.length; i++) { + const table_to_remove = tables_to_remove[i]; + local_db.assign({[table_to_remove]: []}).write(); + logger.debug(`Removed all records from ${table_to_remove}`); + } + return true; + } + + let success = true; + const tables_to_remove = table ? [table] : tables; + logger.debug(`Removing all records from: ${tables_to_remove}`) + for (let i = 0; i < tables_to_remove.length; i++) { + const table_to_remove = tables_to_remove[i]; + + const output = await database.collection(table_to_remove).deleteMany({}); + logger.debug(`Removed all records from ${table_to_remove}`); + success &= !!(output['result']['ok']); + } + return success; +} + +// Stats + +exports.getDBStats = async () => { + const stats_by_table = {}; + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; + if (table === 'test') continue; + + stats_by_table[table] = await getDBTableStats(table); + } + return {stats_by_table: stats_by_table, using_local_db: using_local_db}; +} + +const getDBTableStats = async (table) => { + const table_stats = {}; + // local db override + if (using_local_db) { + table_stats['records_count'] = local_db.get(table).value().length; + } else { + const stats = await database.collection(table).stats(); + table_stats['records_count'] = stats.count; + } + return table_stats; +} + +// JSON to DB + +exports.generateJSONTables = async (db_json, users_json) => { + // create records + let files = db_json['files'] || []; + let playlists = db_json['playlists'] || []; + let categories = db_json['categories'] || []; + let subscriptions = db_json['subscriptions'] || []; + + const users = users_json['users']; + + for (let i = 0; i < users.length; i++) { + const user = users[i]; + + if (user['files']) { + user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] })); + files = files.concat(user['files']); + } + if (user['playlists']) { + user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] })); + playlists = playlists.concat(user['playlists']); + } + if (user['categories']) { + user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] })); + categories = categories.concat(user['categories']); + } + + if (user['subscriptions']) { + user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] })); + subscriptions = subscriptions.concat(user['subscriptions']); + } + } + + const tables_obj = {}; + + // TODO: use create*Records funcs to strip unnecessary properties + tables_obj.files = createFilesRecords(files, subscriptions); + tables_obj.playlists = playlists; + tables_obj.categories = categories; + tables_obj.subscriptions = createSubscriptionsRecords(subscriptions); + tables_obj.users = createUsersRecords(users); + tables_obj.roles = createRolesRecords(users_json['roles']); + tables_obj.downloads = createDownloadsRecords(db_json['downloads']) + + return tables_obj; +} + +exports.importJSONToDB = async (db_json, users_json) => { + // TODO: backup db + + // TODO: delete current records + const tables_obj = await exports.generateJSONTables(db_json, users_json); + + const table_keys = Object.keys(tables_obj); + + let success = true; + for (let i = 0; i < table_keys.length; i++) { + const table_key = table_keys[i]; + success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key]); + } + + return success; +} + +const createFilesRecords = (files, subscriptions) => { + for (let i = 0; i < subscriptions.length; i++) { + const subscription = subscriptions[i]; + subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined})); + files = files.concat(subscriptions[i]['videos']); + console.log(files.length); + } + + return files; +} + +const createPlaylistsRecords = async (playlists) => { + +} + +const createCategoriesRecords = async (categories) => { + +} + +const createSubscriptionsRecords = (subscriptions) => { + for (let i = 0; i < subscriptions.length; i++) { + delete subscriptions[i]['videos']; + } + + return subscriptions; +} + +const createUsersRecords = (users) => { + users.forEach(user => { + delete user['files']; + delete user['playlists']; + delete user['subscriptions']; + }); + return users; +} + +const createRolesRecords = (roles) => { + const new_roles = []; + Object.keys(roles).forEach(role_key => { + new_roles.push({ + key: role_key, + ...roles[role_key] + }); + }); + return new_roles; +} + +const createDownloadsRecords = (downloads) => { + const new_downloads = []; + Object.keys(downloads).forEach(session_key => { + new_downloads.push({ + key: session_key, + ...downloads[session_key] + }); + }); + return new_downloads; +} + +exports.transferDB = async (local_to_remote) => { + const table_to_records = {}; + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; + table_to_records[table] = await exports.getRecords(table); + } + + using_local_db = !local_to_remote; + if (local_to_remote) { + // backup local DB + logger.debug('Backup up Local DB...'); + await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`); + const db_connected = await exports.connectToDB(5, true); + if (!db_connected) { + logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.'); + return false; + } + } + success = true; + + logger.debug('Clearing new database before transfer...'); + + await exports.removeAllRecords(); + + logger.debug('Database cleared! Beginning transfer.'); + + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; + if (!table_to_records[table] || table_to_records[table].length === 0) continue; + success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]); + } + + config_api.setConfigItem('ytdl_use_local_db', using_local_db); + + return success; +} + +/* + This function is necessary to emulate mongodb's ability to search for null or missing values. + A filter of null or undefined for a property will find docs that have that property missing, or have it + null or undefined. We want that same functionality for the local DB as well +*/ +const applyFilterLocalDB = (db_path, filter_obj, operation) => { + const filter_props = Object.keys(filter_obj); + const return_val = db_path[operation](record => { + if (!filter_props) return true; + let filtered = true; + for (let i = 0; i < filter_props.length; i++) { + const filter_prop = filter_props[i]; + const filter_prop_value = filter_obj[filter_prop]; + if (filter_prop_value === undefined || filter_prop_value === null) { + filtered &= record[filter_prop] === undefined || record[filter_prop] === null + } else { + filtered &= record[filter_prop] === filter_prop_value; + } + } + return filtered; + }); + return return_val; +} \ No newline at end of file diff --git a/backend/package-lock.json b/backend/package-lock.json index e7edcd2..36b0d89 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -441,6 +441,11 @@ "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" + }, "buffer": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz", @@ -923,6 +928,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -2042,6 +2052,12 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -2196,6 +2212,44 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "mongodb": { + "version": "3.6.9", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.9.tgz", + "integrity": "sha512-1nSCKgSunzn/CXwgOWgbPHUWOO5OfERcuOWISmqd610jn0s8BU9K4879iJVabqgpPPbA6hO7rG48eq+fGED3Mg==", + "requires": { + "bl": "^2.2.1", + "bson": "^1.1.4", + "denque": "^1.4.1", + "optional-require": "^1.0.3", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + }, + "dependencies": { + "bl": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", + "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + } + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -2423,6 +2477,11 @@ "mimic-fn": "^2.1.0" } }, + "optional-require": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.3.tgz", + "integrity": "sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA==" + }, "p-cancelable": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", @@ -2820,6 +2879,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -2930,6 +2998,15 @@ "is-arrayish": "^0.3.1" } }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", diff --git a/backend/package.json b/backend/package.json index 72607de..b2cd7c1 100644 --- a/backend/package.json +++ b/backend/package.json @@ -5,7 +5,8 @@ "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "nodemon app.js" + "start": "nodemon app.js", + "debug": "set YTDL_MODE=debug && node app.js" }, "nodemonConfig": { "ignore": [ @@ -46,6 +47,7 @@ "merge-files": "^0.1.2", "mocha": "^8.4.0", "moment": "^2.29.1", + "mongodb": "^3.6.9", "multer": "^1.4.2", "node-fetch": "^2.6.1", "node-id3": "^0.1.14", diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 8f29cae..cd15f40 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -14,13 +14,13 @@ const debugMode = process.env.YTDL_MODE === 'debug'; var logger = null; var db = null; var users_db = null; -var db_api = null; +let db_api = null; -function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } +function setDB(input_db_api) { db_api = input_db_api } function setLogger(input_logger) { logger = input_logger; } -function initialize(input_db, input_users_db, input_logger, input_db_api) { - setDB(input_db, input_users_db, input_db_api); +function initialize(input_db_api, input_logger) { + setDB(input_db_api); setLogger(input_logger); } @@ -34,12 +34,7 @@ async function subscribe(sub, user_uid = null) { sub.isPlaylist = sub.url.includes('playlist'); sub.videos = []; - let url_exists = false; - - if (user_uid) - url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value() - else - url_exists = !!db.get('subscriptions').find({url: sub.url}).value(); + let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid})); if (!sub.name && url_exists) { logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`); @@ -48,19 +43,12 @@ async function subscribe(sub, user_uid = null) { return; } - // add sub to db - let sub_db = null; - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write(); - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - } else { - db.get('subscriptions').push(sub).write(); - sub_db = db.get('subscriptions').find({id: sub.id}); - } + sub['user_uid'] = user_uid ? user_uid : undefined; + await db_api.insertRecordIntoTable('subscriptions', sub); + let success = await getSubscriptionInfo(sub, user_uid); if (success) { - sub = sub_db.value(); getVideosForSub(sub, user_uid); } else { logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') @@ -91,8 +79,8 @@ async function getSubscriptionInfo(sub, user_uid = null) { } } - return new Promise(resolve => { - youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, function(err, output) { + return new Promise(async resolve => { + youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => { if (debugMode) { logger.info('Subscribe: got info for subscription ' + sub.id); } @@ -122,10 +110,7 @@ async function getSubscriptionInfo(sub, user_uid = null) { } // if it's now valid, update if (sub.name) { - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); - else - db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); + await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name}); } } @@ -141,10 +126,8 @@ async function getSubscriptionInfo(sub, user_uid = null) { // updates subscription sub.archive = archive_dir; - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); - else - db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); + + await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir}); } // TODO: get even more info @@ -166,10 +149,8 @@ async function unsubscribe(sub, deleteMode, user_uid = null) { let result_obj = { success: false, error: '' }; let id = sub.id; - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write(); - else - db.get('subscriptions').remove({id: id}).write(); + await db_api.removeRecord('subscriptions', {id: id}); + await db_api.removeAllRecords('files', {sub_id: id}); // failed subs have no name, on unsubscribe they shouldn't error if (!sub.name) { @@ -191,20 +172,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) { } async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { + // TODO: combine this with deletefile let basePath = null; - let sub_db = null; - if (user_uid) { - basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - } else { - basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - sub_db = db.get('subscriptions').find({id: sub.id}); - } + basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions') + : config_api.getConfigItem('ytdl_subscriptions_base_path'); const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const appendedBasePath = getAppendedBasePath(sub, basePath); const name = file; let retrievedID = null; - sub_db.get('videos').remove({uid: file_uid}).write(); + + await db_api.removeRecord('files', {uid: file_uid}); let filePath = appendedBasePath; const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' @@ -255,14 +232,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, } async function getVideosForSub(sub, user_uid = null) { - // get sub_db - let sub_db = null; - if (user_uid) - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - else - sub_db = db.get('subscriptions').find({id: sub.id}); - - const latest_sub_obj = sub_db.value(); + const latest_sub_obj = await getSubscription(sub.id); if (!latest_sub_obj || latest_sub_obj['downloading']) { return false; } @@ -292,12 +262,12 @@ async function getVideosForSub(sub, user_uid = null) { // get videos logger.verbose('Subscription: getting videos for subscription ' + sub.name); - return new Promise(resolve => { + return new Promise(async resolve => { const preimported_file_paths = []; const PREIMPORT_INTERVAL = 5000; - const preregister_check = setInterval(() => { + const preregister_check = setInterval(async () => { if (sub.streamingOnly) return; - db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath); + await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath); }, PREIMPORT_INTERVAL); youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { // cleanup @@ -313,7 +283,7 @@ async function getVideosForSub(sub, user_uid = null) { const outputs = err.stdout.split(/\r\n|\r|\n/); for (let i = 0; i < outputs.length; i++) { const output = JSON.parse(outputs[i]); - handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) + await handleOutputJSON(sub, output, i === 0, multiUserMode) if (err.stderr.includes(output['id']) && archive_path) { // we found a video that errored! add it to the archive to prevent future errors if (sub.archive) { @@ -347,7 +317,7 @@ async function getVideosForSub(sub, user_uid = null) { } const reset_videos = i === 0; - handleOutputJSON(sub, sub_db, output_json, multiUserMode, preimported_file_paths, reset_videos); + await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos); } if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { @@ -444,8 +414,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de return downloadConfig; } -function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { - if (sub.streamingOnly) { +async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) { + // TODO: remove streaming only mode + if (false && sub.streamingOnly) { if (reset_videos) { sub_db.assign({videos: []}).write(); } @@ -459,12 +430,15 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ path_object = path.parse(output_json['_filename']); const path_string = path.format(path_object); - if (sub_db.get('videos').find({path: path_string}).value()) { + const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id}); + if (file_exists) { + // TODO: fix issue where files of different paths due to custom path get downloaded multiple times // file already exists in DB, return early to avoid reseting the download date return; } - db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); + await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id); + const url = output_json['webpage_url']; if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) { @@ -477,73 +451,41 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ } } -function getSubscriptions(user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').value(); - else - return db.get('subscriptions').value(); +async function getSubscriptions(user_uid = null) { + return await db_api.getRecords('subscriptions', {user_uid: user_uid}); } -function getAllSubscriptions() { - let subscriptions = null; +async function getAllSubscriptions() { + const all_subs = await db_api.getRecords('subscriptions'); const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode) { - subscriptions = []; - let users = users_db.get('users').value(); - for (let i = 0; i < users.length; i++) { - if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); - } - } else { - subscriptions = getSubscriptions(); - } - return subscriptions; + return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode); } -function getSubscription(subID, user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); - else - return db.get('subscriptions').find({id: subID}).value(); +async function getSubscription(subID) { + return await db_api.getRecord('subscriptions', {id: subID}); } -function getSubscriptionByName(subName, user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value(); - else - return db.get('subscriptions').find({name: subName}).value(); +async function getSubscriptionByName(subName, user_uid = null) { + return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid}); } -function updateSubscription(sub, user_uid = null) { - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write(); - } else { - db.get('subscriptions').find({id: sub.id}).assign(sub).write(); - } +async function updateSubscription(sub, user_uid = null) { + await db_api.updateRecord('subscriptions', {id: sub.id}, sub); return true; } -function updateSubscriptionPropertyMultiple(subs, assignment_obj) { - subs.forEach(sub => { - updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); +async function updateSubscriptionPropertyMultiple(subs, assignment_obj) { + subs.forEach(async sub => { + await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); }); } -function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); - } else { - db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); - } +async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { + // TODO: combine with updateSubscription + await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj); return true; } -function subExists(subID, user_uid = null) { - if (user_uid) - return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); - else - return !!db.get('subscriptions').find({id: subID}).value(); -} - async function setFreshUploads(sub, user_uid) { const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); sub.videos.forEach(async video => { @@ -559,7 +501,7 @@ async function checkVideosForFreshUploads(sub, user_uid) { const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); sub.videos.forEach(async video => { if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) { - checkVideoIfBetterExists(video, sub, user_uid) + await checkVideoIfBetterExists(video, sub, user_uid) } }); } @@ -569,7 +511,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path); logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); // simulate a download to verify that a better version exists - youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => { + youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => { if (err) { // video is not available anymore for whatever reason } else if (output) { diff --git a/backend/test/tests.js b/backend/test/tests.js index 9697a36..52d5260 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -29,7 +29,7 @@ const logger = winston.createLogger({ // new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), new winston.transports.File({ filename: 'appdata/logs/combined.log' }), - new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'}) + new winston.transports.Console({level: 'debug', name: 'console'}) ] }); @@ -38,19 +38,167 @@ var db_api = require('../db'); const utils = require('../utils'); const subscriptions_api = require('../subscriptions'); const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); db_api.initialize(db, users_db, logger); -auth_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); + + +describe('Database', async function() { + describe('Import', async function() { + it('Migrate', async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords(); + const success = await db_api.importJSONToDB(db.value(), users_db.value()); + assert(success); + }); + + it('Transfer to remote', async function() { + await db_api.removeAllRecords('test'); + await db_api.insertRecordIntoTable('test', {test: 'test'}); + + await db_api.transferDB(true); + const success = await db_api.getRecord('test', {test: 'test'}); + assert(success); + }); + + it('Transfer to local', async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('test'); + await db_api.insertRecordIntoTable('test', {test: 'test'}); + + await db_api.transferDB(false); + const success = await db_api.getRecord('test', {test: 'test'}); + assert(success); + }); + }); + + describe('Export', function() { + + }); + + describe('Import and Export', async function() { + it('Existing data', async function() { + const users_db_json = users_db.value(); + const db_json = db.value(); + + const users_db_json_stringified = JSON.stringify(users_db_json); + const db_json_stringified = JSON.stringify(db_json); + + const tables_obj = await db_api.importJSONtoDB(users_db_json, db_json); + const db_jsons = await db_api.exportDBToJSON(tables_obj); + + const users_db_json_returned_stringified = db_jsons['users_db_json']; + const db_json_returned_stringified = db_jsons['db_json']; + + assert(users_db_json_returned_stringified.length === users_db_json_stringified.length); + assert(db_json_returned_stringified.length === db_json_stringified.length); + }); + }); + + describe('Basic functions', async function() { + beforeEach(async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('test'); + }); + it('Add and read record', async function() { + await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); + const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); + assert(added_record['test_add'] === 'test'); + await db_api.removeRecord('test', {test_add: 'test'}); + }); + + it('Update record', async function() { + await db_api.insertRecordIntoTable('test', {test_update: 'test'}); + await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); + const updated_record = await db_api.getRecord('test', {test_update: 'test'}); + assert(updated_record['added_field']); + await db_api.removeRecord('test', {test_update: 'test'}); + }); + + it('Remove record', async function() { + await db_api.insertRecordIntoTable('test', {test_remove: 'test'}); + const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'}); + assert(delete_succeeded); + const deleted_record = await db_api.getRecord('test', {test_remove: 'test'}); + assert(!deleted_record); + }); + + it('Push to record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []}); + await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 1); + }); + + it('Pull from record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']}); + await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 0); + }); + + it('Bulk add', async function() { + const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 + const test_records = []; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + test_records.push({ + uid: uuid() + }); + } + const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records); + + const received_records = await db_api.getRecords('test'); + assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); + }); + + it('Bulk update', async function() { + // bulk add records + const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 + const test_records = []; + const update_obj = {}; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + const test_uid = uuid(); + test_records.push({ + uid: test_uid + }); + update_obj[test_uid] = {added_field: true}; + } + let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); + assert(success); + + // makes sure they are added + const received_records = await db_api.getRecords('test'); + assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); + + success = await db_api.bulkUpdateRecords('test', 'uid', update_obj); + assert(success); + + const received_updated_records = await db_api.getRecords('test'); + for (let i = 0; i < received_updated_records.length; i++) { + success &= received_updated_records[i]['added_field']; + } + assert(success); + }); + + it('Stats', async function() { + const stats = await db_api.getDBStats(); + assert(stats); + }); + }); +}); describe('Multi User', async function() { let user = null; const user_to_test = 'admin'; const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; const playlist_to_test = 'ysabVZz4x'; - before(async function() { + beforeEach(async function() { + await db_api.connectToDB(); + auth_api.initialize(db_api, logger); + subscriptions_api.initialize(db_api, logger); user = await auth_api.login('admin', 'pass'); - console.log('hi') }); describe('Authentication', function() { it('login', async function() { @@ -93,11 +241,12 @@ describe('Multi User', async function() { }); it('Subscription zip generator', async function() { - const sub = subscriptions_api.getSubscription(sub_to_test, user_to_test); + const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test); + const sub_videos = await db_api.getRecords('files', {sub_id: sub.id}); assert(sub); const sub_files_to_download = []; - for (let i = 0; i < sub['videos'].length; i++) { - const sub_file = sub['videos'][i]; + for (let i = 0; i < sub_videos.length; i++) { + const sub_file = sub_videos[i]; sub_files_to_download.push(sub_file); } const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download); diff --git a/backend/utils.js b/backend/utils.js index 880fb9c..dc425e8 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -122,6 +122,21 @@ function getJSONMp3(name, customPath, openReadPerms = false) { return obj; } +function getJSON(file_path, type) { + const ext = type === 'audio' ? '.mp3' : '.mp4'; + let obj = null; + var jsonPath = removeFileExtension(file_path) + '.info.json'; + var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`; + if (fs.existsSync(jsonPath)) + { + obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + } else if (fs.existsSync(alternateJsonPath)) { + obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); + } + else obj = 0; + return obj; +} + function getJSONByType(type, name, customPath, openReadPerms = false) { return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) } @@ -143,6 +158,23 @@ function getDownloadedThumbnail(name, type, customPath = null) { return null; } +function getDownloadedThumbnail2(file_path, type) { + const file_path_no_extension = removeFileExtension(file_path); + + let jpgPath = file_path_no_extension + '.jpg'; + let webpPath = file_path_no_extension + '.webp'; + let pngPath = file_path_no_extension + '.png'; + + if (fs.existsSync(jpgPath)) + return jpgPath; + else if (fs.existsSync(webpPath)) + return webpPath; + else if (fs.existsSync(pngPath)) + return pngPath; + else + return null; +} + function getExpectedFileSize(input_info_jsons) { // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons]; @@ -190,6 +222,28 @@ function fixVideoMetadataPerms(name, type, customPath = null) { } } +function fixVideoMetadataPerms2(file_path, type) { + if (is_windows) return; + + const ext = type === 'audio' ? '.mp3' : '.mp4'; + + const file_path_no_extension = removeFileExtension(file_path); + + const files_to_fix = [ + // JSONs + file_path_no_extension + '.info.json', + file_path_no_extension + ext + '.info.json', + // Thumbnails + file_path_no_extension + '.webp', + file_path_no_extension + '.jpg' + ]; + + for (const file of files_to_fix) { + if (!fs.existsSync(file)) continue; + fs.chmodSync(file, 0o644); + } +} + function deleteJSONFile(name, type, customPath = null) { if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path'); @@ -202,6 +256,18 @@ function deleteJSONFile(name, type, customPath = null) { if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); } +function deleteJSONFile2(file_path, type) { + const ext = type === 'audio' ? '.mp3' : '.mp4'; + + const file_path_no_extension = removeFileExtension(file_path); + + let json_path = file_path_no_extension + '.info.json'; + let alternate_json_path = file_path_no_extension + ext + '.info.json'; + + if (fs.existsSync(json_path)) fs.unlinkSync(json_path); + if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); +} + async function removeIDFromArchive(archive_path, id) { let data = await fs.readFile(archive_path, {encoding: 'utf-8'}); if (!data) { @@ -309,11 +375,15 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p module.exports = { getJSONMp3: getJSONMp3, getJSONMp4: getJSONMp4, + getJSON: getJSON, getTrueFileName: getTrueFileName, getDownloadedThumbnail: getDownloadedThumbnail, + getDownloadedThumbnail2: getDownloadedThumbnail2, getExpectedFileSize: getExpectedFileSize, fixVideoMetadataPerms: fixVideoMetadataPerms, + fixVideoMetadataPerms2: fixVideoMetadataPerms2, deleteJSONFile: deleteJSONFile, + deleteJSONFile2: deleteJSONFile2, removeIDFromArchive, removeIDFromArchive, getDownloadedFilesByType: getDownloadedFilesByType, createContainerZipFile: createContainerZipFile, diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html index 8880b30..c6870b5 100644 --- a/src/app/components/downloads/downloads.component.html +++ b/src/app/components/downloads/downloads.component.html @@ -1,21 +1,21 @@
-
- +
+ -

Session ID: {{session_downloads.key}} -  (current) +

Session ID: {{session_downloads['session_id']}} +  (current)

-
- - +
+ +
- +
diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index d7e7d2a..1539448 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -35,7 +35,7 @@ import { Router } from '@angular/router'; export class DownloadsComponent implements OnInit, OnDestroy { downloads_check_interval = 1000; - downloads = {}; + downloads = []; interval_id = null; keys = Object.keys; @@ -137,6 +137,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { this.downloads[session_id] = session_downloads_by_id; } else { for (let j = 0; j < session_download_ids.length; j++) { + if (session_download_ids[j] === 'session_id' || session_download_ids[j] === '_id') continue; const download_id = session_download_ids[j]; const download = new_downloads_by_session[session_id][download_id] if (!this.downloads[session_id][download_id]) { @@ -156,11 +157,10 @@ export class DownloadsComponent implements OnInit, OnDestroy { downloadsValid() { let valid = false; - const keys = this.keys(this.downloads); - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; - const value = this.downloads[key]; - if (this.keys(value).length > 0) { + for (let i = 0; i < this.downloads.length; i++) { + const session_downloads = this.downloads[i]; + if (!session_downloads) continue; + if (this.keys(session_downloads).length > 2) { valid = true; break; } diff --git a/src/app/components/manage-role/manage-role.component.html b/src/app/components/manage-role/manage-role.component.html index b3e8de9..047dc20 100644 --- a/src/app/components/manage-role/manage-role.component.html +++ b/src/app/components/manage-role/manage-role.component.html @@ -5,7 +5,7 @@

{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}

- + Yes No diff --git a/src/app/components/manage-role/manage-role.component.ts b/src/app/components/manage-role/manage-role.component.ts index 4e05e29..dbe2511 100644 --- a/src/app/components/manage-role/manage-role.component.ts +++ b/src/app/components/manage-role/manage-role.component.ts @@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit { } changeRolePermissions(change, permission) { - this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => { + this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => { if (res['success']) { } else { diff --git a/src/app/components/modify-users/modify-users.component.html b/src/app/components/modify-users/modify-users.component.html index 31c2b80..cd054af 100644 --- a/src/app/components/modify-users/modify-users.component.html +++ b/src/app/components/modify-users/modify-users.component.html @@ -94,7 +94,7 @@
- +
diff --git a/src/app/components/modify-users/modify-users.component.ts b/src/app/components/modify-users/modify-users.component.ts index 9e54fda..e1b7a98 100644 --- a/src/app/components/modify-users/modify-users.component.ts +++ b/src/app/components/modify-users/modify-users.component.ts @@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit { getRoles() { this.postsService.getRoles().subscribe(res => { - this.roles = []; - const roles = res['roles']; - const role_names = Object.keys(roles); - for (let i = 0; i < role_names.length; i++) { - const role_name = role_names[i]; - this.roles.push({ - name: role_name, - permissions: roles[role_name]['permissions'] - }); - } + this.roles = res['roles']; }); } diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.ts b/src/app/dialogs/modify-playlist/modify-playlist.component.ts index 161cab8..b482bb8 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts @@ -61,7 +61,7 @@ export class ModifyPlaylistComponent implements OnInit { } playlistChanged() { - return JSON.stringify(this.playlist) === JSON.stringify(this.original_playlist); + return JSON.stringify(this.playlist) !== JSON.stringify(this.original_playlist); } getPlaylist() { diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index f3a1090..2d4ddb0 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -20,11 +20,16 @@ - - - {{option.label}} + + Max - + + + + {{option.key}} + + +
- + - + diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 0d319ab..a554031 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -171,7 +171,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { console.error('Failed to increment view count'); console.error(err); }); - this.uids = this.db_file['uid']; + this.uids = [this.db_file['uid']]; this.show_player = true; this.parseFileNames(); } @@ -304,12 +304,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { } downloadContent() { - const fileNames = []; - for (let i = 0; i < this.playlist.length; i++) { - fileNames.push(this.playlist[i].title); - } - - const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0]; + const zipName = this.db_playlist.name; this.downloading = true; this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => { this.downloading = false; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 0abeffa..11ff640 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -184,6 +184,18 @@ export class PostsService implements CanActivate { cropFileSettings: cropFileSettings}, this.httpOptions); } + getDBInfo() { + return this.http.post(this.path + 'getDBInfo', {}, this.httpOptions); + } + + transferDB(local_to_remote) { + return this.http.post(this.path + 'transferDB', {local_to_remote: local_to_remote}, this.httpOptions); + } + + testConnectionString() { + return this.http.post(this.path + 'testConnectionString', {}, this.httpOptions); + } + killAllDownloads() { return this.http.post(this.path + 'killAllDownloads', {}, this.httpOptions); } @@ -236,11 +248,12 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions); } - downloadFileFromServer(uid, uuid = null, sub_id = null) { + downloadFileFromServer(uid, uuid = null, sub_id = null, is_playlist = null) { return this.http.post(this.path + 'downloadFileFromServer', { uid: uid, uuid: uuid, - sub_id: sub_id + sub_id: sub_id, + is_playlist: is_playlist }, {responseType: 'blob', params: this.httpOptions.params}); } diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 346797c..9d3d72f 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -280,6 +280,43 @@
+ + + +
+
+
+
+
Database Info
+

Database location: {{db_info['using_local_db'] ? 'Local' : 'MongoDB'}}

+
Records per table
+ + + {{table_stats.key}}: {{table_stats.value.records_count}} + + + + + + Example: mongodb://127.0.0.1:27017/?compressors=zlib + + +
+ + + +
+ + +
+
+ Database information could not be retrieved. Check the server logs for more information. +
+
+
+
+ + diff --git a/src/app/settings/settings.component.scss b/src/app/settings/settings.component.scss index f85952f..aba8baf 100644 --- a/src/app/settings/settings.component.scss +++ b/src/app/settings/settings.component.scss @@ -77,8 +77,13 @@ } .category-custom-placeholder { -background: #ccc; -border: dotted 3px #999; -min-height: 60px; -transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); + background: #ccc; + border: dotted 3px #999; + min-height: 60px; + transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); +} + +.transfer-db-button { + margin-top: 10px; + margin-bottom: 10px; } \ No newline at end of file diff --git a/src/app/settings/settings.component.ts b/src/app/settings/settings.component.ts index 0537103..c0e6399 100644 --- a/src/app/settings/settings.component.ts +++ b/src/app/settings/settings.component.ts @@ -29,6 +29,10 @@ export class SettingsComponent implements OnInit { generated_bookmarklet_code = null; bookmarkletAudioOnly = false; + db_info = null; + db_transferring = false; + testing_connection_string = false; + _settingsSame = true; latestGithubRelease = null; @@ -48,6 +52,7 @@ export class SettingsComponent implements OnInit { ngOnInit() { this.getConfig(); + this.getDBInfo(); this.generated_bookmarklet_code = this.sanitizer.bypassSecurityTrustUrl(this.generateBookmarkletCode()); @@ -263,6 +268,60 @@ export class SettingsComponent implements OnInit { }); } + getDBInfo() { + this.postsService.getDBInfo().subscribe(res => { + this.db_info = res['db_info']; + }); + } + + transferDB() { + const dialogRef = this.dialog.open(ConfirmDialogComponent, { + data: { + dialogTitle: 'Transfer DB', + dialogText: `Are you sure you want to transfer the DB?`, + submitText: 'Transfer', + } + }); + dialogRef.afterClosed().subscribe(confirmed => { + if (confirmed) { + this._transferDB(); + } + }); + } + + _transferDB() { + this.db_transferring = true; + this.postsService.transferDB(this.db_info['using_local_db']).subscribe(res => { + this.db_transferring = false; + const success = res['success']; + if (success) { + this.openSnackBar('Successfully transfered DB! Reloading info...'); + this.getDBInfo(); + } else { + this.openSnackBar('Failed to transfer DB -- transfer was aborted. Error: ' + res['error']); + } + }, err => { + this.db_transferring = false; + this.openSnackBar('Failed to transfer DB -- API call failed. See browser logs for details.'); + console.error(err); + }); + } + + testConnectionString() { + this.testing_connection_string = true; + this.postsService.testConnectionString().subscribe(res => { + this.testing_connection_string = false; + if (res['success']) { + this.postsService.openSnackBar('Connection successful!'); + } else { + this.postsService.openSnackBar('Connection failed! Error: ' + res['error']); + } + }, err => { + this.testing_connection_string = false; + this.postsService.openSnackBar('Connection failed! Error: Server error. See logs for more info.'); + }); + } + // snackbar helper public openSnackBar(message: string, action: string = '') { this.snackBar.open(message, action, {