diff --git a/Dockerfile b/Dockerfile index 75b22d3..68d648d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,9 @@ ENV UID=1000 \ GID=1000 \ USER=youtube +ENV NO_UPDATE_NOTIFIER=true +ENV FOREVER_ROOT=/app/.forever + RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID RUN apk add --no-cache \ @@ -33,6 +36,7 @@ RUN apk add --no-cache \ WORKDIR /app COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ] +RUN npm install forever -g RUN npm install && chown -R $UID:$GID ./ COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] @@ -40,4 +44,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ] EXPOSE 17442 ENTRYPOINT [ "/app/entrypoint.sh" ] -CMD [ "node", "app.js" ] +CMD [ "forever", "app.js" ] diff --git a/README.md b/README.md index c073161..1c4dcae 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker Debian/Ubuntu: ```bash -sudo apt-get install nodejs youtube-dl ffmpeg +sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm ``` CentOS 7: diff --git a/backend/app.js b/backend/app.js index 29781e7..144f33a 100644 --- a/backend/app.js +++ b/backend/app.js @@ -79,9 +79,9 @@ const logger = winston.createLogger({ }); config_api.initialize(logger); -auth_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); +auth_api.initialize(db_api, logger); +subscriptions_api.initialize(db_api, logger); categories_api.initialize(db, users_db, logger, db_api); // Set some defaults @@ -139,18 +139,22 @@ var updaterStatus = null; var timestamp_server_start = Date.now(); +const concurrentStreams = {}; + if (debugMode) logger.info('YTDL-Material in debug mode!'); // check if just updated -const just_restarted = fs.existsSync('restart.json'); -if (just_restarted) { +const just_updated = fs.existsSync('restart_update.json'); +if (just_updated) { updaterStatus = { updating: false, details: 'Update complete! You are now on ' + CONSTS['CURRENT_VERSION'] } - fs.unlinkSync('restart.json'); + fs.unlinkSync('restart_update.json'); } +if (fs.existsSync('restart_general.json')) fs.unlinkSync('restart_general.json'); + // updates & starts youtubedl (commented out b/c of repo takedown) // startYoutubeDL(); @@ -179,7 +183,7 @@ if (writeConfigMode) { loadConfig(); } -var downloads = {}; +var downloads = []; app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); @@ -189,16 +193,6 @@ app.use(auth_api.passport.initialize()); // actual functions -/** - * setTimeout, but its a promise. - * @param {number} ms - */ -async function wait(ms) { - await new Promise(resolve => { - setTimeout(resolve, ms); - }); -} - async function checkMigrations() { // 3.5->3.6 migration const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); @@ -216,14 +210,27 @@ async function checkMigrations() { if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); - success = success && await addMetadataPropertyToDB('view_count'); - success = success && await addMetadataPropertyToDB('description'); - success = success && await addMetadataPropertyToDB('height'); - success = success && await addMetadataPropertyToDB('abr'); + success = success && await db_api.addMetadataPropertyToDB('view_count'); + success = success && await db_api.addMetadataPropertyToDB('description'); + success = success && await db_api.addMetadataPropertyToDB('height'); + success = success && await db_api.addMetadataPropertyToDB('abr'); + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } + const new_db_system_migration_complete = db.get('new_db_system_migration_complete').value(); + if (!new_db_system_migration_complete) { + logger.info('Beginning migration: 4.2->4.3+') + let success = await db_api.importJSONToDB(db.value(), users_db.value()); + + // sets migration to complete + db.set('new_db_system_migration_complete', true).write(); + if (success) { logger.info('4.2->4.3+ migration complete!'); } + else { logger.error('Migration failed: 4.2->4.3+'); } + } + return true; } @@ -237,7 +244,7 @@ async function runFilesToDBMigration() { const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); if (!file_already_in_db) { logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); + db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); } } @@ -246,7 +253,7 @@ async function runFilesToDBMigration() { const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); if (!file_already_in_db) { logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp4', 'video'); + db_api.registerFileDB(file_obj.id + '.mp4', 'video'); } } @@ -296,28 +303,6 @@ async function simplifyDBFileStructure() { return true; } -async function addMetadataPropertyToDB(property_key) { - try { - const dirs_to_check = db_api.getFileDirectoriesAndDBs(); - for (const dir_to_check of dirs_to_check) { - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); - for (const file of files) { - if (file[property_key]) { - dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); - } - } - } - - // sets migration to complete - db.set('simplified_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; - } -} - async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku @@ -332,18 +317,12 @@ async function startServer() { }); } -async function restartServer() { - const restartProcess = () => { - spawn('node', ['app.js'], { - detached: true, - stdio: 'inherit' - }).unref() - process.exit() - } - logger.info('Update complete! Restarting server...'); +async function restartServer(is_update = false) { + logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`); // the following line restarts the server through nodemon - fs.writeFileSync('restart.json', 'internal use only'); + fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only'); + process.exit(1); } async function updateServer(tag) { @@ -386,7 +365,7 @@ async function updateServer(tag) { updating: true, 'details': 'Update complete! Restarting server...' } - restartServer(); + restartServer(true); }, err => { updaterStatus = { updating: false, @@ -531,7 +510,7 @@ async function backupServerLite() { }); // wait a tiny bit for the zip to reload in fs - await wait(100); + await utils.wait(100); return true; } @@ -599,7 +578,7 @@ async function killAllDownloads() { async function setPortItemFromENV() { config_api.setConfigItem('ytdl_port', backendPort.toString()); - await wait(100); + await utils.wait(100); return true; } @@ -613,7 +592,7 @@ async function setConfigFromEnv() { let success = config_api.setConfigItems(config_items); if (success) { logger.info('Config items set using ENV variables.'); - await wait(100); + await utils.wait(100); return true; } else { logger.error('ERROR: Failed to set config items using ENV variables.'); @@ -624,6 +603,9 @@ async function setConfigFromEnv() { async function loadConfig() { loadConfigValues(); + // connect to DB + await db_api.connectToDB(); + // creates archive path if missing await fs.ensureDir(archivePath); @@ -636,7 +618,7 @@ async function loadConfig() { // get subscriptions if (allowSubscriptions) { // set downloading to false - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds watchSubscriptions(); @@ -645,10 +627,10 @@ async function loadConfig() { }, subscriptionsCheckInterval * 1000); } - db_api.importUnregisteredFiles(); + await db_api.importUnregisteredFiles(); // load in previous downloads - downloads = db.get('downloads').value(); + downloads = await db_api.getRecords('downloads'); // start the server here startServer(); @@ -696,7 +678,7 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) { } async function watchSubscriptions() { - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; @@ -849,199 +831,6 @@ function getVideoFormatID(name) } } -async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { - let zipFolderPath = null; - - if (!fullPathProvided) { - zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath); - if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); - } else { - zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - - let ext = (type === 'audio') ? '.mp3' : '.mp4'; - - let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip')); - - var archive = archiver('zip', { - gzip: true, - zlib: { level: 9 } // Sets the compression level. - }); - - archive.on('error', function(err) { - logger.error(err); - throw err; - }); - - // pipe archive data to the output file - archive.pipe(output); - - for (let i = 0; i < fileNames.length; i++) { - let fileName = fileNames[i]; - let fileNamePathRemoved = path.parse(fileName).base; - let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName; - archive.file(file_path, {name: fileNamePathRemoved + ext}) - } - - await archive.finalize(); - - // wait a tiny bit for the zip to reload in fs - await wait(100); - return path.join(zipFolderPath,outputName + '.zip'); -} - -async function deleteAudioFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : audioFolderPath; - - var jsonPath = path.join(filePath,name+'.mp3.info.json'); - var altJSONPath = path.join(filePath,name+'.info.json'); - var audioFilePath = path.join(filePath,name+'.mp3'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - altJSONPath = path.join(__dirname, altJSONPath); - audioFilePath = path.join(__dirname, audioFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - let audioFileExists = await fs.pathExists(audioFilePath); - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_audio.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp3(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('audio', line); - } else { - logger.info('Could not find archive file for audio files. Creating...'); - await fs.close(await fs.open(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (audioFileExists) { - await fs.unlink(audioFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(audioFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -async function deleteVideoFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : videoFolderPath; - var jsonPath = path.join(filePath,name+'.info.json'); - - var altJSONPath = path.join(filePath,name+'.mp4.info.json'); - var videoFilePath = path.join(filePath,name+'.mp4'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - videoFilePath = path.join(__dirname, videoFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let videoFileExists = await fs.pathExists(videoFilePath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp4(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('video', line); - } else { - logger.info('Could not find archive file for videos. Creating...'); - fs.closeSync(fs.openSync(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (videoFileExists) { - await fs.unlink(videoFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(videoFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - /** * @param {'audio' | 'video'} type * @param {string[]} fileNames @@ -1108,8 +897,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // adds download to download helper const download_uid = uuid(); const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { + let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session); + if (!session_downloads) { + session_downloads = {session_id: session}; + downloads.push(session_downloads); + } + session_downloads[download_uid] = { uid: download_uid, ui_uid: options.ui_uid, downloading: true, @@ -1121,7 +914,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { timestamp_start: Date.now(), filesize: null }; - const download = downloads[session][download_uid]; + const download = session_downloads[download_uid]; updateDownloads(); let download_checker = null; @@ -1133,7 +926,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { return; } else if (info) { // check if it fits into a category. If so, then get info again using new downloadConfig - if (!Array.isArray(info)) category = await categories_api.categorize(info); + if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); // set custom output if the category has one and re-retrieve info so the download manager has the right file name if (category && category['custom_output']) { @@ -1155,12 +948,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } // download file - youtubedl.exec(url, downloadConfig, {}, function(err, output) { + youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) download['downloading'] = false; download['timestamp_end'] = Date.now(); - var file_uid = null; + var file_objs = []; let new_date = Date.now(); let difference = (new_date - date)/1000; logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); @@ -1227,10 +1020,17 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; - // registers file in DB - file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category); + if (options.cropFileSettings) { + await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext); + } + // registers file in DB + const file_obj = await db_api.registerFileDB2(full_file_path, options.user, category, null, options.cropFileSettings); + + // TODO: remove the following line if (file_name) file_names.push(file_name); + + file_objs.push(file_obj); } let is_playlist = file_names.length > 1; @@ -1246,158 +1046,28 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { download['fileNames'] = is_playlist ? file_names : [full_file_path] updateDownloads(); - var videopathEncoded = encodeURIComponent(file_names[0]); + let container = null; + + if (file_objs.length > 1) { + // create playlist + const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); + const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); + container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, file_objs[0]['thumbnailURL'], options.user); + } else if (file_objs.length === 1) { + container = file_objs[0]; + } else { + logger.error('Downloaded file failed to result in metadata object.'); + } resolve({ - [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: is_playlist ? file_names : null, - uid: file_uid + file_uids: file_objs.map(file_obj => file_obj.uid), + container: container }); } }); }); } -async function downloadFileByURL_normal(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - var file_uid = null; - const is_audio = type === 'audio'; - const ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; } - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'normal'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - const video = youtubedl(url, - // Optional arguments passed to youtube-dl. - downloadConfig, - // Additional options can be given for calling `child_process.execFile()`. - { cwd: __dirname }); - - let video_info = null; - let file_size = 0; - - // Will be called when the download starts. - video.on('info', function(info) { - video_info = info; - file_size = video_info.size; - const json_path = utils.removeFileExtension(video_info._filename) + '.info.json'; - fs.ensureFileSync(json_path); - fs.writeJSONSync(json_path, video_info); - video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) - }); - // Will be called if download was already completed and there is nothing more to download. - video.on('complete', function complete(info) { - 'use strict' - logger.info('file ' + info._filename + ' already downloaded.') - }) - - let download_pos = 0; - video.on('data', function data(chunk) { - download_pos += chunk.length - // `size` should not be 0 here. - if (file_size) { - let percent = (download_pos / file_size * 100).toFixed(2) - download['percent_complete'] = percent; - } - }); - - video.on('end', async function() { - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - download['timestamp_end'] = Date.now(); - download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext]; - download['complete'] = true; - updateDownloads(); - - // audio-only cleanup - if (is_audio) { - // filename fix - video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3'; - - // ID3 tagging - let tags = { - title: video_info['title'], - artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] - } - let success = NodeID3.write(tags, video_info._filename); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); - - const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm'; - const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4'; - // check if audio file is webm - if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']); - else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']); - } - - // registers file in DB - const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name)); - - resolve({ - [is_audio ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready - uid: file_uid - }); - }); - - video.on('error', function error(err) { - logger.error(err); - - download[error] = err; - updateDownloads(); - - resolve(false); - }); - }); - -} - async function generateArgs(url, type, options) { var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; var globalArgs = config_api.getConfigItem('ytdl_custom_args'); @@ -1438,8 +1108,8 @@ async function generateArgs(url, type, options) { qualityPath = ['-f', customQualityConfiguration]; } else if (selectedHeight && selectedHeight !== '' && !is_audio) { qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; - } else if (maxBitrate && is_audio) { - qualityPath = ['--audio-quality', maxBitrate] + } else if (is_audio) { + qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0'] } if (customOutput) { @@ -1565,7 +1235,7 @@ async function getUrlInfos(urls) { let startDate = Date.now(); let result = []; return new Promise(resolve => { - youtubedl.exec(urls.join(' '), ['--dump-json'], {}, (err, output) => { + youtubedl.exec(urls.join(' '), ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { let new_date = Date.now(); let difference = (new_date - startDate)/1000; logger.debug(`URL info retrieval delay: ${difference} seconds.`); @@ -1587,34 +1257,49 @@ async function getUrlInfos(urls) { }); } -async function convertFileToMp3(input_file, output_file) { - logger.verbose(`Converting ${input_file} to ${output_file}...`); +// ffmpeg helper functions + +async function cropFile(file_path, start, end, ext) { return new Promise(resolve => { - ffmpeg(input_file).noVideo().toFormat('mp3') - .on('end', () => { - logger.verbose(`Conversion for '${output_file}' complete.`); - fs.unlinkSync(input_file) - resolve(true); - }) - .on('error', (err) => { - logger.error('Failed to convert audio file to the correct format.'); - logger.error(err); - resolve(false); - }).save(output_file); - }); + const temp_file_path = `${file_path}.cropped${ext}`; + let base_ffmpeg_call = ffmpeg(file_path); + if (start) { + base_ffmpeg_call = base_ffmpeg_call.seekOutput(start); + } + if (end) { + base_ffmpeg_call = base_ffmpeg_call.duration(end - start); + } + base_ffmpeg_call + .on('end', () => { + logger.verbose(`Cropping for '${file_path}' complete.`); + fs.unlinkSync(file_path); + fs.moveSync(temp_file_path, file_path); + resolve(true); + }) + .on('error', (err, test, test2) => { + logger.error(`Failed to crop ${file_path}.`); + logger.error(err); + resolve(false); + }).save(temp_file_path); + }); } +// archive helper functions + async function writeToBlacklist(type, line) { let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); // adds newline to the beginning of the line + line.replace('\n', ''); + line.replace('\r', ''); line = '\n' + line; await fs.appendFile(blacklistPath, line); } // download management functions -function updateDownloads() { - db.assign({downloads: downloads}).write(); +async function updateDownloads() { + await db_api.removeAllRecords('downloads'); + if (downloads.length !== 0) await db_api.insertRecordsIntoTable('downloads', downloads); } function checkDownloadPercent(download) { @@ -1646,7 +1331,6 @@ function checkDownloadPercent(download) { } }); download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2); - updateDownloads(); }); } @@ -1834,18 +1518,18 @@ app.use(function(req, res, next) { app.use(compression()); -const optionalJwt = function (req, res, next) { +const optionalJwt = async function (req, res, next) { const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); if (multiUserMode && ((req.body && req.body.uuid) || (req.query && req.query.uuid)) && (req.path.includes('/api/getFile') || req.path.includes('/api/stream') || - req.path.includes('/api/downloadFile'))) { + req.path.includes('/api/getPlaylist') || + req.path.includes('/api/downloadFileFromServer'))) { // check if shared video const using_body = req.body && req.body.uuid; const uuid = using_body ? req.body.uuid : req.query.uuid; const uid = using_body ? req.body.uid : req.query.uid; - const type = using_body ? req.body.type : req.query.type; - const playlist_id = using_body ? req.body.id : req.query.id; - const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, type, true, req.body) : auth_api.getUserPlaylist(uuid, playlist_id, null, false); + const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id; + const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true); if (file) { req.can_watch = true; return next(); @@ -1883,41 +1567,48 @@ app.post('/api/setConfig', optionalJwt, function(req, res) { logger.error('Tried to save invalid config file!') res.sendStatus(400); } - }); -app.post('/api/tomp3', optionalJwt, async function(req, res) { - var url = req.body.url; - var options = { - customArgs: req.body.customArgs, - customOutput: req.body.customOutput, - maxBitrate: req.body.maxBitrate, - customQualityConfiguration: req.body.customQualityConfiguration, - youtubeUsername: req.body.youtubeUsername, - youtubePassword: req.body.youtubePassword, - ui_uid: req.body.ui_uid, - user: req.isAuthenticated() ? req.user.uid : null - } +app.post('/api/restartServer', optionalJwt, (req, res) => { + restartServer(); + res.send({success: true}); +}); - const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload(); - if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.'); - const is_playlist = url.includes('playlist'); +app.post('/api/getDBInfo', optionalJwt, async (req, res) => { + const db_info = await db_api.getDBStats(); + res.send({db_info: db_info}); +}); - let result_obj = null; - if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate) - result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID); - else - result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID); - if (result_obj) { - res.send(result_obj); +app.post('/api/transferDB', optionalJwt, async (req, res) => { + const local_to_remote = req.body.local_to_remote; + let success = null; + let error = ''; + if (local_to_remote === config_api.getConfigItem('ytdl_use_local_db')) { + success = await db_api.transferDB(local_to_remote); + if (!success) error = 'Unknown error'; + else config_api.setConfigItem('ytdl_use_local_db', !local_to_remote); } else { - res.sendStatus(500); + success = false; + error = `Failed to transfer DB as it cannot transition into its current status: ${local_to_remote ? 'MongoDB' : 'Local DB'}`; + logger.error(error); } + + res.send({success: success, error: error}); }); -app.post('/api/tomp4', optionalJwt, async function(req, res) { +app.post('/api/testConnectionString', optionalJwt, async (req, res) => { + let success = null; + let error = ''; + success = await db_api.connectToDB(5, true); + if (!success) error = 'Connection string failed.'; + + res.send({success: success, error: error}); +}); + +app.post('/api/downloadFile', optionalJwt, async function(req, res) { req.setTimeout(0); // remove timeout in case of long videos - var url = req.body.url; + const url = req.body.url; + const type = req.body.type; var options = { customArgs: req.body.customArgs, customOutput: req.body.customOutput, @@ -1926,18 +1617,11 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) { youtubeUsername: req.body.youtubeUsername, youtubePassword: req.body.youtubePassword, ui_uid: req.body.ui_uid, - user: req.isAuthenticated() ? req.user.uid : null + user: req.isAuthenticated() ? req.user.uid : null, + cropFileSettings: req.body.cropFileSettings } - const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload(); - if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.'); - const is_playlist = url.includes('playlist'); - - let result_obj = null; - if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu')) - result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID); - else - result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID); + let result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID); if (result_obj) { res.send(result_obj); } else { @@ -1950,29 +1634,17 @@ app.post('/api/killAllDownloads', optionalJwt, async function(req, res) { res.send(result_obj); }); -/** - * add thumbnails if present - * @param files - List of files with thumbnailPath property. - */ -async function addThumbnails(files) { - await Promise.all(files.map(async file => { - const thumbnailPath = file['thumbnailPath']; - if (thumbnailPath && (await fs.pathExists(thumbnailPath))) { - file['thumbnailBlob'] = await fs.readFile(thumbnailPath); - } - })); -} - // gets all download mp3s app.get('/api/getMp3s', optionalJwt, async function(req, res) { - var mp3s = db.get('files').value().filter(file => file.isAudio === true); - var playlists = db.get('playlists').value(); + // TODO: simplify + let mp3s = await db_api.getRecords('files', {isAudio: true}); + let playlists = await db_api.getRecords('playlists'); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user audio files/playlists auth_api.passport.authenticate('jwt') - mp3s = auth_api.getUserVideos(req.user.uid, 'audio'); - playlists = auth_api.getUserPlaylists(req.user.uid); + mp3s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: true}); + playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove? } mp3s = JSON.parse(JSON.stringify(mp3s)); @@ -1985,15 +1657,15 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { // gets all download mp4s app.get('/api/getMp4s', optionalJwt, async function(req, res) { - var mp4s = db.get('files').value().filter(file => file.isAudio === false); - var playlists = db.get('playlists').value(); + let mp4s = await db_api.getRecords('files', {isAudio: false}); + let playlists = await db_api.getRecords('playlists'); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user videos/playlists auth_api.passport.authenticate('jwt') - mp4s = auth_api.getUserVideos(req.user.uid, 'video'); - playlists = auth_api.getUserPlaylists(req.user.uid); + mp4s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: false}); + playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove? } mp4s = JSON.parse(JSON.stringify(mp4s)); @@ -2004,20 +1676,14 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) { }); }); -app.post('/api/getFile', optionalJwt, function (req, res) { +app.post('/api/getFile', optionalJwt, async function (req, res) { var uid = req.body.uid; var type = req.body.type; var uuid = req.body.uuid; - var file = null; + var file = await db_api.getRecord('files', {uid: uid}); - if (req.isAuthenticated()) { - file = auth_api.getUserVideo(req.user.uid, uid); - } else if (uuid) { - file = auth_api.getUserVideo(uuid, uid, true); - } else { - file = db.get('files').find({uid: uid}).value(); - } + if (uuid && !file['sharingEnabled']) file = null; // check if chat exists for twitch videos if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json'); @@ -2038,43 +1704,16 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { // these are returned let files = null; let playlists = null; + const uuid = req.isAuthenticated() ? req.user.uid : null; - let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; + let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (await subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; - // get basic info depending on multi-user mode being enabled - if (req.isAuthenticated()) { - files = auth_api.getUserVideos(req.user.uid); - playlists = auth_api.getUserPlaylists(req.user.uid, files); - } else { - files = db.get('files').value(); - playlists = JSON.parse(JSON.stringify(db.get('playlists').value())); - const categories = db.get('categories').value(); - if (categories) { - categories.forEach(category => { - const audio_files = files && files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); - const video_files = files && files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); - if (audio_files && audio_files.length > 0) { - playlists.push({ - name: category['name'], - thumbnailURL: audio_files[0].thumbnailURL, - thumbnailPath: audio_files[0].thumbnailPath, - fileNames: audio_files.map(file => file.id), - type: 'audio', - auto: true - }); - } - if (video_files && video_files.length > 0) { - playlists.push({ - name: category['name'], - thumbnailURL: video_files[0].thumbnailURL, - thumbnailPath: video_files[0].thumbnailPath, - fileNames: video_files.map(file => file.id), - type: 'video', - auto: true - }); - } - }); - } + files = await db_api.getRecords('files', {user_uid: uuid}); + playlists = await db_api.getRecords('playlists', {user_uid: uuid}); + + const categories = await categories_api.getCategoriesAsPlaylists(files); + if (categories) { + playlists = playlists.concat(categories); } // loop through subscriptions and add videos @@ -2091,17 +1730,40 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { files = JSON.parse(JSON.stringify(files)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(files); - } - res.send({ files: files, playlists: playlists }); }); +app.post('/api/checkConcurrentStream', async (req, res) => { + const uid = req.body.uid; + + const DEAD_SERVER_THRESHOLD = 10; + + if (concurrentStreams[uid] && Date.now()/1000 - concurrentStreams[uid]['unix_timestamp'] > DEAD_SERVER_THRESHOLD) { + logger.verbose( `Killing dead stream on ${uid}`); + delete concurrentStreams[uid]; + } + + res.send({stream: concurrentStreams[uid]}) +}); + +app.post('/api/updateConcurrentStream', optionalJwt, async (req, res) => { + const uid = req.body.uid; + const playback_timestamp = req.body.playback_timestamp; + const unix_timestamp = req.body.unix_timestamp; + const playing = req.body.playing; + + concurrentStreams[uid] = { + playback_timestamp: playback_timestamp, + unix_timestamp: unix_timestamp, + playing: playing + } + + res.send({stream: concurrentStreams[uid]}) +}); + app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => { var id = req.body.id; var type = req.body.type; @@ -2143,7 +1805,7 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { }); // video sharing -app.post('/api/enableSharing', optionalJwt, function(req, res) { +app.post('/api/enableSharing', optionalJwt, async (req, res) => { var uid = req.body.uid; var is_playlist = req.body.is_playlist; let success = false; @@ -2158,17 +1820,11 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { // single-user mode try { success = true; - if (!is_playlist && type !== 'subscription') { - db.get(`files`) - .find({uid: uid}) - .assign({sharingEnabled: true}) - .write(); + if (!is_playlist) { + await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: true}) } else if (is_playlist) { - db.get(`playlists`) - .find({id: uid}) - .assign({sharingEnabled: true}) - .write(); - } else if (type === 'subscription') { + await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true}); + } else if (false) { // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // time they are requested from the subscription directory. } else { @@ -2177,6 +1833,7 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { } } catch(err) { + logger.error(err); success = false; } @@ -2185,7 +1842,7 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { }); }); -app.post('/api/disableSharing', optionalJwt, function(req, res) { +app.post('/api/disableSharing', optionalJwt, async function(req, res) { var type = req.body.type; var uid = req.body.uid; var is_playlist = req.body.is_playlist; @@ -2202,15 +1859,9 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files`) - .find({uid: uid}) - .assign({sharingEnabled: false}) - .write(); + await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false}) } else if (is_playlist) { - db.get(`playlists`) - .find({id: uid}) - .assign({sharingEnabled: false}) - .write(); + await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false}); } else if (type === 'subscription') { // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // time they are requested from the subscription directory. @@ -2252,7 +1903,7 @@ app.post('/api/incrementViewCount', optionalJwt, async (req, res) => { // categories app.post('/api/getAllCategories', optionalJwt, async (req, res) => { - const categories = db.get('categories').value(); + const categories = await db_api.getRecords('categories'); res.send({categories: categories}); }); @@ -2262,10 +1913,10 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { name: name, uid: uuid(), rules: [], - custom_putput: '' + custom_output: '' }; - db.get('categories').push(new_category).write(); + await db_api.insertRecordIntoTable('categories', new_category); res.send({ new_category: new_category, @@ -2276,7 +1927,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { app.post('/api/deleteCategory', optionalJwt, async (req, res) => { const category_uid = req.body.category_uid; - db.get('categories').remove({uid: category_uid}).write(); + await db_api.removeRecord('categories', {uid: category_uid}); res.send({ success: true @@ -2285,13 +1936,14 @@ app.post('/api/deleteCategory', optionalJwt, async (req, res) => { app.post('/api/updateCategory', optionalJwt, async (req, res) => { const category = req.body.category; - db.get('categories').find({uid: category.uid}).assign(category).write(); + await db_api.updateRecord('categories', {uid: category.uid}, category) res.send({success: true}); }); app.post('/api/updateCategories', optionalJwt, async (req, res) => { const categories = req.body.categories; - db.get('categories').assign(categories).write(); + await db_api.removeAllRecords('categories'); + await db_api.insertRecordsIntoTable('categories', categories); res.send({success: true}); }); @@ -2390,9 +2042,9 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { // get sub from db let subscription = null; if (subID) { - subscription = subscriptions_api.getSubscription(subID, user_uid) + subscription = await subscriptions_api.getSubscription(subID, user_uid) } else if (subName) { - subscription = subscriptions_api.getSubscriptionByName(subName, user_uid) + subscription = await subscriptions_api.getSubscriptionByName(subName, user_uid) } if (!subscription) { @@ -2403,7 +2055,8 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { // get sub videos if (subscription.name && !subscription.streamingOnly) { - var parsed_files = subscription.videos; + var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos; + subscription['videos'] = parsed_files; if (!parsed_files) { parsed_files = []; let base_path = null; @@ -2500,7 +2153,7 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { let user_uid = req.isAuthenticated() ? req.user.uid : null; // get subs from api - let subscriptions = subscriptions_api.getSubscriptions(user_uid); + let subscriptions = await subscriptions_api.getSubscriptions(user_uid); res.send({ subscriptions: subscriptions @@ -2509,29 +2162,11 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { app.post('/api/createPlaylist', optionalJwt, async (req, res) => { let playlistName = req.body.playlistName; - let fileNames = req.body.fileNames; + let uids = req.body.uids; let type = req.body.type; let thumbnailURL = req.body.thumbnailURL; - let duration = req.body.duration; - - let new_playlist = { - name: playlistName, - fileNames: fileNames, - id: shortid.generate(), - thumbnailURL: thumbnailURL, - type: type, - registered: Date.now(), - duration: duration - }; - - if (req.isAuthenticated()) { - auth_api.addPlaylist(req.user.uid, new_playlist, type); - } else { - db.get(`playlists`) - .push(new_playlist) - .write(); - } + const new_playlist = await db_api.createPlaylist(playlistName, uids, type, thumbnailURL, req.isAuthenticated() ? req.user.uid : null); res.send({ new_playlist: new_playlist, @@ -2540,19 +2175,25 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { }); app.post('/api/getPlaylist', optionalJwt, async (req, res) => { - let playlistID = req.body.playlistID; - let uuid = req.body.uuid; + let playlist_id = req.body.playlist_id; + let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null); + let include_file_metadata = req.body.include_file_metadata; - let playlist = null; + const playlist = await db_api.getPlaylist(playlist_id, uuid); + const file_objs = []; - if (req.isAuthenticated()) { - playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID); - } else { - playlist = db.get(`playlists`).find({id: playlistID}).value(); + if (playlist && include_file_metadata) { + for (let i = 0; i < playlist['uids'].length; i++) { + const uid = playlist['uids'][i]; + const file_obj = await db_api.getVideo(uid, uuid); + if (file_obj) file_objs.push(file_obj); + // TODO: remove file from playlist if could not be found + } } res.send({ playlist: playlist, + file_objs: file_objs, type: playlist && playlist.type, success: !!playlist }); @@ -2560,17 +2201,14 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; - let fileNames = req.body.fileNames; + let uids = req.body.uids; let success = false; try { if (req.isAuthenticated()) { - auth_api.updatePlaylistFiles(req.user.uid, playlistID, fileNames); + auth_api.updatePlaylistFiles(req.user.uid, playlistID, uids); } else { - db.get(`playlists`) - .find({id: playlistID}) - .assign({fileNames: fileNames}) - .write(); + await db_api.updateRecord('playlists', {id: playlistID}, {uids: uids}) } success = true; @@ -2585,7 +2223,7 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { let playlist = req.body.playlist; - let success = db_api.updatePlaylist(playlist, req.user && req.user.uid); + let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid); res.send({ success: success }); @@ -2596,14 +2234,8 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { let success = null; try { - if (req.isAuthenticated()) { - auth_api.removePlaylist(req.user.uid, playlistID); - } else { - // removes playlist from playlists - db.get(`playlists`) - .remove({id: playlistID}) - .write(); - } + // removes playlist from playlists + await db_api.removeRecord('playlists', {id: playlistID}) success = true; } catch(e) { @@ -2617,82 +2249,61 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { // deletes non-subscription files app.post('/api/deleteFile', optionalJwt, async (req, res) => { - var uid = req.body.uid; - var type = req.body.type; - var blacklistMode = req.body.blacklistMode; + const uid = req.body.uid; + const blacklistMode = req.body.blacklistMode; + const uuid = req.isAuthenticated() ? req.user.uid : null; - if (req.isAuthenticated()) { - let success = await auth_api.deleteUserFile(req.user.uid, uid, blacklistMode); - res.send(success); - return; - } - - var file_obj = db.get(`files`).find({uid: uid}).value(); - var name = file_obj.id; - var fullpath = file_obj ? file_obj.path : null; - var wasDeleted = false; - if (await fs.pathExists(fullpath)) - { - wasDeleted = type === 'audio' ? await deleteAudioFile(name, path.dirname(fullpath), blacklistMode) : await deleteVideoFile(name, path.dirname(fullpath), blacklistMode); - db.get('files').remove({uid: uid}).write(); - wasDeleted = true; - res.send(wasDeleted); - } else if (file_obj) { - db.get('files').remove({uid: uid}).write(); - wasDeleted = true; - res.send(wasDeleted); - } else { - wasDeleted = false; - res.send(wasDeleted); - } + let wasDeleted = false; + wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode); + res.send(wasDeleted); }); -app.post('/api/downloadFile', optionalJwt, async (req, res) => { - let fileNames = req.body.fileNames; - let zip_mode = req.body.zip_mode; - let type = req.body.type; - let outputName = req.body.outputName; - let fullPathProvided = req.body.fullPathProvided; - let subscriptionName = req.body.subscriptionName; - let subscriptionPlaylist = req.body.subPlaylist; - let file = null; - if (!zip_mode) { - fileNames = decodeURIComponent(fileNames); - const is_audio = type === 'audio'; - const fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - const ext = is_audio ? '.mp3' : '.mp4'; +app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => { + let uid = req.body.uid; + let uuid = req.body.uuid; + let playlist_id = req.body.playlist_id; + let sub_id = req.body.sub_id; - let base_path = fileFolderPath; - let usersFileFolder = null; - const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode && (req.body.uuid || req.user.uid)) { - usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - base_path = path.join(usersFileFolder, req.body.uuid ? req.body.uuid : req.user.uid, type); - } - if (!subscriptionName) { - file = path.join(__dirname, base_path, fileNames + ext); - } else { - let basePath = null; - if (usersFileFolder) - basePath = path.join(usersFileFolder, req.user.uid, 'subscriptions'); - else - basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + let file_path_to_download = null; - file = path.join(__dirname, basePath, (subscriptionPlaylist === true || subscriptionPlaylist === 'true' ? 'playlists' : 'channels'), subscriptionName, fileNames + ext); + if (!uuid && req.user) uuid = req.user.uid; + + let zip_file_generated = false; + if (playlist_id) { + zip_file_generated = true; + const playlist_files_to_download = []; + const playlist = await db_api.getPlaylist(playlist_id, uuid); + for (let i = 0; i < playlist['uids'].length; i++) { + const playlist_file_uid = playlist['uids'][i]; + const file_obj = await db_api.getVideo(playlist_file_uid, uuid); + playlist_files_to_download.push(file_obj); } + + // generate zip + file_path_to_download = await utils.createContainerZipFile(playlist, playlist_files_to_download); + } else if (sub_id && !uid) { + zip_file_generated = true; + const sub_files_to_download = []; + const sub = subscriptions_api.getSubscription(sub_id, uuid); + for (let i = 0; i < sub['videos'].length; i++) { + const sub_file = sub['videos'][i]; + sub_files_to_download.push(sub_file); + } + + // generate zip + file_path_to_download = await utils.createContainerZipFile(sub, sub_files_to_download); } else { - for (let i = 0; i < fileNames.length; i++) { - fileNames[i] = decodeURIComponent(fileNames[i]); - } - file = await createPlaylistZipFile(fileNames, type, outputName, fullPathProvided, req.body.uuid); - if (!path.isAbsolute(file)) file = path.join(__dirname, file); + const file_obj = await db_api.getVideo(uid, uuid, sub_id) + file_path_to_download = file_obj.path; } - res.sendFile(file, function (err) { + if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download); + res.sendFile(file_path_to_download, function (err) { if (err) { logger.error(err); - } else if (fullPathProvided) { + } else if (zip_file_generated) { try { - fs.unlinkSync(file); + // delete generated zip file + fs.unlinkSync(file_path_to_download); } catch(e) { logger.error("Failed to remove file", file); } @@ -2767,31 +2378,23 @@ app.post('/api/generateNewAPIKey', function (req, res) { // Streaming API calls -app.get('/api/stream/:id', optionalJwt, (req, res) => { +app.get('/api/stream', optionalJwt, async (req, res) => { const type = req.query.type; + const uuid = req.query.uuid ? req.query.uuid : (req.user ? req.user.uid : null); + const sub_id = req.query.sub_id; const ext = type === 'audio' ? '.mp3' : '.mp4'; const mimetype = type === 'audio' ? 'audio/mp3' : 'video/mp4'; var head; let optionalParams = url_api.parse(req.url,true).query; - let id = decodeURIComponent(req.params.id); - let file_path = req.query.file_path ? decodeURIComponent(req.query.file_path.split('?')[0]) : null; - if (!file_path && (req.isAuthenticated() || req.can_watch)) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - if (optionalParams['subName']) { - const isPlaylist = optionalParams['subPlaylist']; - file_path = path.join(usersFileFolder, req.user.uid, 'subscriptions', (isPlaylist === 'true' ? 'playlists/' : 'channels/'),optionalParams['subName'], id + ext) - } else { - file_path = path.join(usersFileFolder, req.query.uuid ? req.query.uuid : req.user.uid, type, id + ext); - } - } else if (!file_path && optionalParams['subName']) { - let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const isPlaylist = optionalParams['subPlaylist']; - basePath += (isPlaylist === 'true' ? 'playlists/' : 'channels/'); - file_path = basePath + optionalParams['subName'] + '/' + id + ext; - } + let uid = decodeURIComponent(req.query.uid); - if (!file_path) { - file_path = path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); + let file_path = null; + + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + if (!multiUserMode || req.isAuthenticated() || req.can_watch) { + const file_obj = await db_api.getVideo(uid, uuid, sub_id); + if (file_obj) file_path = file_obj['path']; + else file_path = null; } const stat = fs.statSync(file_path) @@ -2805,11 +2408,11 @@ app.get('/api/stream/:id', optionalJwt, (req, res) => { : fileSize-1 const chunksize = (end-start)+1 const file = fs.createReadStream(file_path, {start, end}) - if (config_api.descriptors[id]) config_api.descriptors[id].push(file); - else config_api.descriptors[id] = [file]; + if (config_api.descriptors[uid]) config_api.descriptors[uid].push(file); + else config_api.descriptors[uid] = [file]; file.on('close', function() { - let index = config_api.descriptors[id].indexOf(file); - config_api.descriptors[id].splice(index, 1); + let index = config_api.descriptors[uid].indexOf(file); + config_api.descriptors[uid].splice(index, 1); logger.debug('Successfully closed stream and removed file reference.'); }); head = { @@ -2843,20 +2446,14 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { }); app.post('/api/download', async (req, res) => { - var session_id = req.body.session_id; - var download_id = req.body.download_id; + const session_id = req.body.session_id; + const download_id = req.body.download_id; + const session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session_id); let found_download = null; // find download - if (downloads[session_id] && Object.keys(downloads[session_id])) { - let session_downloads = Object.values(downloads[session_id]); - for (let i = 0; i < session_downloads.length; i++) { - let session_download = session_downloads[i]; - if (session_download && session_download['ui_uid'] === download_id) { - found_download = session_download; - break; - } - } + if (session_downloads && Object.keys(session_downloads)) { + found_download = Object.values(session_downloads).find(session_download => session_download['ui_uid'] === download_id); } if (found_download) { @@ -2874,26 +2471,22 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { var download_id = req.body.download_id; if (delete_all) { // delete all downloads - downloads = {}; + downloads = []; success = true; } else if (download_id) { // delete just 1 download - if (downloads[session_id][download_id]) { - delete downloads[session_id][download_id]; + const session_downloads = downloads.find(session => session['session_id'] === session_id); + if (session_downloads && session_downloads[download_id]) { + delete session_downloads[download_id]; success = true; - } else if (!downloads[session_id]) { + } else if (!session_downloads) { logger.error(`Session ${session_id} has no downloads.`) - } else if (!downloads[session_id][download_id]) { + } else if (!session_downloads[download_id]) { logger.error(`Download '${download_id}' for session '${session_id}' could not be found`); } } else if (session_id) { // delete a session's downloads - if (downloads[session_id]) { - delete downloads[session_id]; - success = true; - } else { - logger.error(`Session ${session_id} has no downloads.`) - } + downloads = downloads.filter(session => session['session_id'] !== session_id); } updateDownloads(); res.send({success: success, downloads: downloads}); @@ -2978,29 +2571,28 @@ app.post('/api/auth/changePassword', optionalJwt, async (req, res) => { res.send({success: success}); }); app.post('/api/auth/adminExists', async (req, res) => { - let exists = auth_api.adminExists(); + let exists = await auth_api.adminExists(); res.send({exists: exists}); }); // user management app.post('/api/getUsers', optionalJwt, async (req, res) => { - let users = users_db.get('users').value(); + let users = await db_api.getRecords('users'); res.send({users: users}); }); app.post('/api/getRoles', optionalJwt, async (req, res) => { - let roles = users_db.get('roles').value(); + let roles = await db_api.getRecords('roles'); res.send({roles: roles}); }); app.post('/api/updateUser', optionalJwt, async (req, res) => { let change_obj = req.body.change_object; try { - const user_db_obj = users_db.get('users').find({uid: change_obj.uid}); if (change_obj.name) { - user_db_obj.assign({name: change_obj.name}).write(); + await db_api.updateRecord('users', {uid: change_obj.uid}, {name: change_obj.name}); } if (change_obj.role) { - user_db_obj.assign({role: change_obj.role}).write(); + await db_api.updateRecord('users', {uid: change_obj.uid}, {role: change_obj.role}); } res.send({success: true}); } catch (err) { @@ -3012,13 +2604,17 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => { app.post('/api/deleteUser', optionalJwt, async (req, res) => { let uid = req.body.uid; try { + let success = false; let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const user_folder = path.join(__dirname, usersFileFolder, uid); - const user_db_obj = users_db.get('users').find({uid: uid}); - if (user_db_obj.value()) { + const user_db_obj = await db_api.getRecord('users', {uid: uid}); + if (user_db_obj) { // user exists, let's delete await fs.remove(user_folder); - users_db.get('users').remove({uid: uid}).write(); + await db_api.removeRecord('users', {uid: uid}); + success = true; + } else { + logger.error(`Could not find user with uid ${uid}`); } res.send({success: true}); } catch (err) { @@ -3037,7 +2633,7 @@ app.post('/api/changeUserPermissions', optionalJwt, async (req, res) => { return; } - const success = auth_api.changeUserPermissions(user_uid, permission, new_value); + const success = await auth_api.changeUserPermissions(user_uid, permission, new_value); res.send({success: success}); }); @@ -3052,7 +2648,7 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => { return; } - const success = auth_api.changeRolePermissions(role, permission, new_value); + const success = await auth_api.changeRolePermissions(role, permission, new_value); res.send({success: success}); }); diff --git a/backend/appdata/default.json b/backend/appdata/default.json index dca9eee..5c2ba9c 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -20,7 +20,8 @@ "allow_quality_select": true, "download_only_mode": false, "allow_multi_download_mode": true, - "enable_downloads_manager": true + "enable_downloads_manager": true, + "allow_playlist_categorization": true }, "API": { "use_API_key": false, @@ -53,6 +54,10 @@ "searchFilter": "(uid={{username}})" } }, + "Database": { + "use_local_db": false, + "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" + }, "Advanced": { "default_downloader": "youtube-dl", "use_default_downloading_agent": true, diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index e7cf337..7992aac 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -1,12 +1,10 @@ const path = require('path'); const config_api = require('../config'); const consts = require('../consts'); -var subscriptions_api = require('../subscriptions') const fs = require('fs-extra'); -var jwt = require('jsonwebtoken'); +const jwt = require('jsonwebtoken'); const { uuid } = require('uuidv4'); -var bcrypt = require('bcryptjs'); - +const bcrypt = require('bcryptjs'); var LocalStrategy = require('passport-local').Strategy; var LdapStrategy = require('passport-ldapauth'); @@ -15,16 +13,15 @@ var JwtStrategy = require('passport-jwt').Strategy, // other required vars let logger = null; -let db = null; -let users_db = null; +let db_api = null; let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function(input_db, input_users_db, input_logger) { +exports.initialize = function(db_api, input_logger) { setLogger(input_logger) - setDB(input_db, input_users_db); + setDB(db_api); /************************* * Authentication module @@ -34,21 +31,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) { JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration'); SERVER_SECRET = null; - if (users_db.get('jwt_secret').value()) { - SERVER_SECRET = users_db.get('jwt_secret').value(); + if (db_api.users_db.get('jwt_secret').value()) { + SERVER_SECRET = db_api.users_db.get('jwt_secret').value(); } else { SERVER_SECRET = uuid(); - users_db.set('jwt_secret', SERVER_SECRET).write(); + db_api.users_db.set('jwt_secret', SERVER_SECRET).write(); } opts = {} opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt'); opts.secretOrKey = SERVER_SECRET; - /*opts.issuer = 'example.com'; - opts.audience = 'example.com';*/ - exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) { - const user = users_db.get('users').find({uid: jwt_payload.user}).value(); + exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) { + const user = await db_api.getRecord('users', {uid: jwt_payload.user}); if (user) { return done(null, user); } else { @@ -62,9 +57,8 @@ function setLogger(input_logger) { logger = input_logger; } -function setDB(input_db, input_users_db) { - db = input_db; - users_db = input_users_db; +function setDB(input_db_api) { + db_api = input_db_api; } exports.passport = require('passport'); @@ -80,7 +74,7 @@ exports.passport.deserializeUser(function(user, done) { /*************************************** * Register user with hashed password **************************************/ -exports.registerUser = function(req, res) { +exports.registerUser = async function(req, res) { var userid = req.body.userid; var username = req.body.username; var plaintextPassword = req.body.password; @@ -98,20 +92,20 @@ exports.registerUser = function(req, res) { } bcrypt.hash(plaintextPassword, saltRounds) - .then(function(hash) { + .then(async function(hash) { let new_user = generateUserObject(userid, username, hash); // check if user exists - if (users_db.get('users').find({uid: userid}).value()) { + if (await db_api.getRecord('users', {uid: userid})) { // user id is taken! logger.error('Registration failed: UID is already taken!'); res.status(409).send('UID is already taken!'); - } else if (users_db.get('users').find({name: username}).value()) { + } else if (await db_api.getRecord('users', {name: username})) { // user name is taken! logger.error('Registration failed: User name is already taken!'); res.status(409).send('User name is already taken!'); } else { // add to db - users_db.get('users').push(new_user).write(); + await db_api.insertRecordIntoTable('users', new_user); logger.verbose(`New user created: ${new_user.name}`); res.send({ user: new_user @@ -144,16 +138,18 @@ exports.registerUser = function(req, res) { ************************************************/ +exports.login = async (username, password) => { + const user = await db_api.getRecord('users', {name: username}); + if (!user) { logger.error(`User ${username} not found`); false } + if (user.auth_method && user.auth_method !== 'internal') { return false } + return await bcrypt.compare(password, user.passhash) ? user : false; +} + exports.passport.use(new LocalStrategy({ usernameField: 'username', passwordField: 'password'}, async function(username, password, done) { - const user = users_db.get('users').find({name: username}).value(); - if (!user) { logger.error(`User ${username} not found`); return done(null, false); } - if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); } - if (user) { - return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false); - } + return done(null, await exports.login(username, password)); } )); @@ -164,17 +160,17 @@ var getLDAPConfiguration = function(req, callback) { }; exports.passport.use(new LdapStrategy(getLDAPConfiguration, - function(user, done) { + async function(user, done) { // check if ldap auth is enabled const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap'; if (!ldap_enabled) return done(null, false); const user_uid = user.uid; - let db_user = users_db.get('users').find({uid: user_uid}).value(); + let db_user = await db_api.getRecord('users', {uid: user_uid}); if (!db_user) { // generate DB user let new_user = generateUserObject(user_uid, user_uid, null, 'ldap'); - users_db.get('users').push(new_user).write(); + await db_api.insertRecordIntoTable('users', new_user); db_user = new_user; logger.verbose(`Generated new user ${user_uid} using LDAP`); } @@ -198,11 +194,11 @@ exports.generateJWT = function(req, res, next) { next(); } -exports.returnAuthResponse = function(req, res) { +exports.returnAuthResponse = async function(req, res) { res.status(200).json({ user: req.user, token: req.token, - permissions: exports.userPermissions(req.user.uid), + permissions: await exports.userPermissions(req.user.uid), available_permissions: consts['AVAILABLE_PERMISSIONS'] }); } @@ -215,7 +211,7 @@ exports.returnAuthResponse = function(req, res) { * It also passes the user object to the next * middleware through res.locals **************************************/ -exports.ensureAuthenticatedElseError = function(req, res, next) { +exports.ensureAuthenticatedElseError = (req, res, next) => { var token = getToken(req.query); if( token ) { try { @@ -233,10 +229,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) { } // change password -exports.changeUserPassword = async function(user_uid, new_pass) { +exports.changeUserPassword = async (user_uid, new_pass) => { try { const hash = await bcrypt.hash(new_pass, saltRounds); - users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write(); + await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash}); return true; } catch (err) { return false; @@ -244,16 +240,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) { } // change user permissions -exports.changeUserPermissions = function(user_uid, permission, new_value) { +exports.changeUserPermissions = async (user_uid, permission, new_value) => { try { - const user_db_obj = users_db.get('users').find({uid: user_uid}); - user_db_obj.get('permissions').pull(permission).write(); - user_db_obj.get('permission_overrides').pull(permission).write(); + await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission); + await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); if (new_value === 'yes') { - user_db_obj.get('permissions').push(permission).write(); - user_db_obj.get('permission_overrides').push(permission).write(); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); } else if (new_value === 'no') { - user_db_obj.get('permission_overrides').push(permission).write(); + await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission); } return true; } catch (err) { @@ -263,12 +258,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) { } // change role permissions -exports.changeRolePermissions = function(role, permission, new_value) { +exports.changeRolePermissions = async (role, permission, new_value) => { try { - const role_db_obj = users_db.get('roles').get(role); - role_db_obj.get('permissions').pull(permission).write(); + await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission); if (new_value === 'yes') { - role_db_obj.get('permissions').push(permission).write(); + await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission); } return true; } catch (err) { @@ -277,19 +271,19 @@ exports.changeRolePermissions = function(role, permission, new_value) { } } -exports.adminExists = function() { - return !!users_db.get('users').find({uid: 'admin'}).value(); +exports.adminExists = async function() { + return !!(await db_api.getRecord('users', {uid: 'admin'})); } // video stuff -exports.getUserVideos = function(user_uid, type) { - const user = users_db.get('users').find({uid: user_uid}).value(); - return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files']; +exports.getUserVideos = async function(user_uid, type) { + const files = await db_api.getRecords('files', {user_uid: user_uid}); + return type ? files.filter(file => file.isAudio === (type === 'audio')) : files; } -exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { - let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); +exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) { + let file = await db_api.getRecord('files', {file_uid: file_uid}); // prevent unauthorized users from accessing the file info if (file && !file['sharingEnabled'] && requireSharing) file = null; @@ -297,58 +291,22 @@ exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { return file; } -exports.addPlaylist = function(user_uid, new_playlist) { - users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write(); - return true; -} - exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) { users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames}); return true; } -exports.removePlaylist = function(user_uid, playlistID) { - users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write(); +exports.removePlaylist = async function(user_uid, playlistID) { + await db_api.removeRecord('playlist', {playlistID: playlistID}); return true; } -exports.getUserPlaylists = function(user_uid, user_files = null) { - const user = users_db.get('users').find({uid: user_uid}).value(); - const playlists = JSON.parse(JSON.stringify(user['playlists'])); - const categories = db.get('categories').value(); - if (categories && user_files) { - categories.forEach(category => { - const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); - const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); - if (audio_files && audio_files.length > 0) { - playlists.push({ - name: category['name'], - thumbnailURL: audio_files[0].thumbnailURL, - thumbnailPath: audio_files[0].thumbnailPath, - fileNames: audio_files.map(file => file.id), - type: 'audio', - uid: user_uid, - auto: true - }); - } - if (video_files && video_files.length > 0) { - playlists.push({ - name: category['name'], - thumbnailURL: video_files[0].thumbnailURL, - thumbnailPath: video_files[0].thumbnailPath, - fileNames: video_files.map(file => file.id), - type: 'video', - uid: user_uid, - auto: true - }); - } - }); - } - return playlists; +exports.getUserPlaylists = async function(user_uid, user_files = null) { + return await db_api.getRecords('playlists', {user_uid: user_uid}); } -exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { - let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value(); +exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) { + let playlist = await db_api.getRecord('playlists', {id: playlistID}); // prevent unauthorized users from accessing the file info if (requireSharing && !playlist['sharingEnabled']) playlist = null; @@ -356,109 +314,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) return playlist; } -exports.registerUserFile = function(user_uid, file_object) { - users_db.get('users').find({uid: user_uid}).get(`files`) - .remove({ - path: file_object['path'] - }).write(); - - users_db.get('users').find({uid: user_uid}).get(`files`) - .push(file_object) - .write(); -} - -exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) { +exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) { let success = false; - const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); - if (file_obj) { - const type = file_obj.isAudio ? 'audio' : 'video'; - const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const ext = type === 'audio' ? '.mp3' : '.mp4'; - - // close descriptors - if (config_api.descriptors[file_obj.id]) { - try { - for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) { - config_api.descriptors[file_obj.id][i].destroy(); - } - } catch(e) { - - } - } - - const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext); - users_db.get('users').find({uid: user_uid}).get(`files`) - .remove({ - uid: file_uid - }).write(); - if (await fs.pathExists(full_path)) { - // remove json and file - const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json'); - const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json'); - let youtube_id = null; - if (await fs.pathExists(json_path)) { - youtube_id = await fs.readJSON(json_path).id; - await fs.unlink(json_path); - } else if (await fs.pathExists(alternate_json_path)) { - youtube_id = await fs.readJSON(alternate_json_path).id; - await fs.unlink(alternate_json_path); - } - - await fs.unlink(full_path); - - // do archive stuff - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`); - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null; - if (blacklistMode && line) { - let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`); - // adds newline to the beginning of the line - line = '\n' + line; - await fs.appendFile(blacklistPath, line); - } - } else { - logger.info(`Could not find archive file for ${type} files. Creating...`); - await fs.ensureFile(archive_path); - } - } - } - success = true; - } else { - success = false; - logger.warn(`User file ${file_uid} does not exist!`); - } - + is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled}); + success = true; return success; } -exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) { - let success = false; - const user_db_obj = users_db.get('users').find({uid: user_uid}); - if (user_db_obj.value()) { - const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid}); - if (file_db_obj.value()) { - success = true; - file_db_obj.assign({sharingEnabled: enabled}).write(); - } - } +exports.userHasPermission = async function(user_uid, permission) { - return success; -} - -exports.userHasPermission = function(user_uid, permission) { - const user_obj = users_db.get('users').find({uid: user_uid}).value(); + const user_obj = await db_api.getRecord('users', ({uid: user_uid})); const role = user_obj['role']; if (!role) { // role doesn't exist logger.error('Invalid role ' + role); return false; } - const role_permissions = (users_db.get('roles').value())['permissions']; + const role_permissions = (await db_api.getRecords('roles'))['permissions']; const user_has_explicit_permission = user_obj['permissions'].includes(permission); const permission_in_overrides = user_obj['permission_overrides'].includes(permission); @@ -481,16 +353,17 @@ exports.userHasPermission = function(user_uid, permission) { } } -exports.userPermissions = function(user_uid) { +exports.userPermissions = async function(user_uid) { let user_permissions = []; - const user_obj = users_db.get('users').find({uid: user_uid}).value(); + const user_obj = await db_api.getRecord('users', ({uid: user_uid})); const role = user_obj['role']; if (!role) { // role doesn't exist logger.error('Invalid role ' + role); return null; } - const role_permissions = users_db.get('roles').get(role).get('permissions').value() + const role_obj = await db_api.getRecord('roles', {key: role}); + const role_permissions = role_obj['permissions']; for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) { let permission = consts['AVAILABLE_PERMISSIONS'][i]; diff --git a/backend/categories.js b/backend/categories.js index d0b249a..fb33a88 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -1,4 +1,5 @@ const config_api = require('./config'); +const utils = require('./utils'); var logger = null; var db = null; @@ -33,35 +34,58 @@ Rules: */ -async function categorize(file_json) { +async function categorize(file_jsons) { + // to make the logic easier, let's assume the file metadata is an array + if (!Array.isArray(file_jsons)) file_jsons = [file_jsons]; + let selected_category = null; - const categories = getCategories(); + const categories = await getCategories(); if (!categories) { - logger.warn('Categories could not be found. Initializing categories...'); - db.assign({categories: []}).write(); + logger.warn('Categories could not be found.'); return null; - return; } - for (let i = 0; i < categories.length; i++) { - const category = categories[i]; - const rules = category['rules']; - - // if rules for current category apply, then that is the selected category - if (applyCategoryRules(file_json, rules, category['name'])) { - selected_category = category; - logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); - return selected_category; + for (let i = 0; i < file_jsons.length; i++) { + const file_json = file_jsons[i]; + for (let j = 0; j < categories.length; j++) { + const category = categories[j]; + const rules = category['rules']; + + // if rules for current category apply, then that is the selected category + if (applyCategoryRules(file_json, rules, category['name'])) { + selected_category = category; + logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); + return selected_category; + } } } + return selected_category; } -function getCategories() { - const categories = db.get('categories').value(); +async function getCategories() { + const categories = await db_api.getRecords('categories'); return categories ? categories : null; } +async function getCategoriesAsPlaylists(files = null) { + const categories_as_playlists = []; + const available_categories = await getCategories(); + if (available_categories && files) { + for (category of available_categories) { + const files_that_match = utils.addUIDsToCategory(category, files); + if (files_that_match && files_that_match.length > 0) { + category['thumbnailURL'] = files_that_match[0].thumbnailURL; + category['thumbnailPath'] = files_that_match[0].thumbnailPath; + category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); + category['id'] = category['uid']; + categories_as_playlists.push(category); + } + } + } + return categories_as_playlists; +} + function applyCategoryRules(file_json, rules, category_name) { let rules_apply = false; for (let i = 0; i < rules.length; i++) { @@ -72,10 +96,10 @@ function applyCategoryRules(file_json, rules, category_name) { switch (rule['comparator']) { case 'includes': - rule_applies = file_json[rule['property']].includes(rule['value']); + rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()); break; case 'not_includes': - rule_applies = !(file_json[rule['property']].includes(rule['value'])); + rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase())); break; case 'equals': rule_applies = file_json[rule['property']] === rule['value']; @@ -120,4 +144,6 @@ async function addTagToExistingTags(tag) { module.exports = { initialize: initialize, categorize: categorize, + getCategories: getCategories, + getCategoriesAsPlaylists: getCategoriesAsPlaylists } \ No newline at end of file diff --git a/backend/config.js b/backend/config.js index 4790e34..4a61999 100644 --- a/backend/config.js +++ b/backend/config.js @@ -197,7 +197,8 @@ DEFAULT_CONFIG = { "allow_quality_select": true, "download_only_mode": false, "allow_multi_download_mode": true, - "enable_downloads_manager": true + "enable_downloads_manager": true, + "allow_playlist_categorization": true }, "API": { "use_API_key": false, @@ -230,6 +231,10 @@ DEFAULT_CONFIG = { "searchFilter": "(uid={{username}})" } }, + "Database": { + "use_local_db": false, + "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" + }, "Advanced": { "default_downloader": "youtube-dl", "use_default_downloading_agent": true, diff --git a/backend/consts.js b/backend/consts.js index fa14171..91cfc75 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -68,6 +68,10 @@ let CONFIG_ITEMS = { 'key': 'ytdl_enable_downloads_manager', 'path': 'YoutubeDLMaterial.Extra.enable_downloads_manager' }, + 'ytdl_allow_playlist_categorization': { + 'key': 'ytdl_allow_playlist_categorization', + 'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization' + }, // API 'ytdl_use_api_key': { @@ -149,6 +153,16 @@ let CONFIG_ITEMS = { 'path': 'YoutubeDLMaterial.Users.ldap_config' }, + // Database + 'ytdl_use_local_db': { + 'key': 'ytdl_use_local_db', + 'path': 'YoutubeDLMaterial.Database.use_local_db' + }, + 'ytdl_mongodb_connection_string': { + 'key': 'ytdl_mongodb_connection_string', + 'path': 'YoutubeDLMaterial.Database.mongodb_connection_string' + }, + // Advanced 'ytdl_default_downloader': { 'key': 'ytdl_default_downloader', diff --git a/backend/db.js b/backend/db.js index 6d5cdb6..84032dd 100644 --- a/backend/db.js +++ b/backend/db.js @@ -3,22 +3,141 @@ var path = require('path') var utils = require('./utils') const { uuid } = require('uuidv4'); const config_api = require('./config'); +const { MongoClient } = require("mongodb"); + +const low = require('lowdb') +const FileSync = require('lowdb/adapters/FileSync'); +const local_adapter = new FileSync('./appdata/local_db.json'); +const local_db = low(local_adapter); var logger = null; var db = null; var users_db = null; -function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db } -function setLogger(input_logger) { logger = input_logger; } +var database = null; -function initialize(input_db, input_users_db, input_logger) { +const tables = { + files: { + name: 'files', + primary_key: 'uid' + }, + playlists: { + name: 'playlists', + primary_key: 'id' + }, + categories: { + name: 'categories', + primary_key: 'uid' + }, + subscriptions: { + name: 'subscriptions', + primary_key: 'id' + }, + downloads: { + name: 'downloads' + }, + users: { + name: 'users', + primary_key: 'uid' + }, + roles: { + name: 'roles', + primary_key: 'key' + }, + test: { + name: 'test' + } +} + +const tables_list = Object.keys(tables); + +const local_db_defaults = {} +tables_list.forEach(table => {local_db_defaults[table] = []}); +local_db.defaults(local_db_defaults).write(); + +let using_local_db = config_api.getConfigItem('ytdl_use_local_db'); + +function setDB(input_db, input_users_db) { + db = input_db; users_db = input_users_db; + exports.db = input_db; + exports.users_db = input_users_db +} + +function setLogger(input_logger) { + logger = input_logger; +} + +exports.initialize = (input_db, input_users_db, input_logger) => { setDB(input_db, input_users_db); setLogger(input_logger); } -function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null, category = null) { +exports.connectToDB = async (retries = 5, no_fallback = false) => { + if (using_local_db) return; + const success = await exports._connectToDB(); + if (success) return true; + + logger.warn(`MongoDB connection failed! Retrying ${retries} times...`); + const retry_delay_ms = 2000; + for (let i = 0; i < retries; i++) { + const retry_succeeded = await exports._connectToDB(); + if (retry_succeeded) { + logger.info(`Successfully connected to DB after ${i+1} attempt(s)`); + return true; + } + + if (i !== retries - 1) { + logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`); + await utils.wait(retry_delay_ms); + } else { + logger.warn(`Retry ${i+1} failed.`); + } + } + if (no_fallback) { + logger.error('Failed to connect to MongoDB. Verify your connection string is valid.'); + return; + } + using_local_db = true; + config_api.setConfigItem('ytdl_use_local_db', true); + logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.'); + return true; +} + +exports._connectToDB = async () => { + const uri = config_api.getConfigItem('ytdl_mongodb_connection_string'); // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb"; + const client = new MongoClient(uri, { + useNewUrlParser: true, + useUnifiedTopology: true, + }); + + try { + await client.connect(); + database = client.db('ytdl_material'); + const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name); + + const missing_tables = tables_list.filter(table => !(existing_collections.includes(table))); + missing_tables.forEach(async table => { + await database.createCollection(table); + }); + + tables_list.forEach(async table => { + const primary_key = tables[table]['primary_key']; + if (!primary_key) return; + await database.collection(table).createIndex({[primary_key]: 1}, { unique: true }); + }); + return true; + } catch(err) { + logger.error(err); + return false; + } finally { + // Ensures that the client will close when you finish/error + // await client.close(); + } +} + +exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => { let db_path = null; const file_id = utils.removeFileExtension(file_path); - const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); + if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); if (!file_object) { logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); return false; @@ -32,45 +151,66 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null, custo // if category exists, only include essential info if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; - if (!sub) { - if (multiUserMode) { - const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get(`files`); - } else { - db_path = db.get(`files`); - } - } else { - if (multiUserMode) { - const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).get('videos'); - } else { - db_path = db.get('subscriptions').find({id: sub.id}).get('videos'); - } + // modify duration + if (cropFileSettings) { + file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; } - const file_uid = registerFileDBManual(db_path, file_object); + if (multiUserMode) file_object['user_uid'] = multiUserMode.user; + + const file_obj = await registerFileDBManual(file_object); // remove metadata JSON if needed if (!config_api.getConfigItem('ytdl_include_metadata')) { utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path) } - return file_uid; + return file_obj; } -function registerFileDBManual(db_path, file_object) { +exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { + if (!file_object) file_object = generateFileObject2(file_path, type); + if (!file_object) { + logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); + return false; + } + + utils.fixVideoMetadataPerms2(file_path, type); + + // add thumbnail path + file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type); + + // if category exists, only include essential info + if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; + + // modify duration + if (cropFileSettings) { + file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; + } + + if (user_uid) file_object['user_uid'] = user_uid; + if (sub_id) file_object['sub_id'] = sub_id; + + const file_obj = await registerFileDBManual(file_object); + + // remove metadata JSON if needed + if (!config_api.getConfigItem('ytdl_include_metadata')) { + utils.deleteJSONFile2(file_path, type) + } + + return file_obj; +} + +async function registerFileDBManual(file_object) { // add additional info file_object['uid'] = uuid(); file_object['registered'] = Date.now(); path_object = path.parse(file_object['path']); file_object['path'] = path.format(path_object); - // remove duplicate(s) - db_path.remove({path: file_object['path']}).write(); + exports.insertRecordIntoTable('files', file_object, {path: file_object['path']}) - // add new file to db - db_path.push(file_object).write(); - return file_object['uid']; + return file_object; } function generateFileObject(id, type, customPath = null, sub = null) { @@ -102,23 +242,38 @@ function generateFileObject(id, type, customPath = null, sub = null) { return file_obj; } -function updatePlaylist(playlist, user_uid) { - let playlistID = playlist.id; - let db_loc = null; - if (user_uid) { - db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}); - } else { - db_loc = db.get(`playlists`).find({id: playlistID}); +function generateFileObject2(file_path, type) { + var jsonobj = utils.getJSON(file_path, type); + if (!jsonobj) { + return null; } - db_loc.assign(playlist).write(); - return true; + const ext = (type === 'audio') ? '.mp3' : '.mp4' + const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type); + // console. + var stats = fs.statSync(true_file_path); + + const file_id = utils.removeFileExtension(path.basename(file_path)); + var title = jsonobj.title; + var url = jsonobj.webpage_url; + var uploader = jsonobj.uploader; + var upload_date = jsonobj.upload_date; + upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A'; + + var size = stats.size; + + var thumbnail = jsonobj.thumbnail; + var duration = jsonobj.duration; + var isaudio = type === 'audio'; + var description = jsonobj.description; + var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); + return file_obj; } function getAppendedBasePathSub(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } -function getFileDirectoriesAndDBs() { +exports.getFileDirectoriesAndDBs = async () => { let dirs_to_check = []; let subscriptions_to_check = []; const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode @@ -126,48 +281,45 @@ function getFileDirectoriesAndDBs() { const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions'); if (multi_user_mode) { - let users = users_db.get('users').value(); + const users = await exports.getRecords('users'); for (let i = 0; i < users.length; i++) { const user = users[i]; - if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']); - // add user's audio dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'audio'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files'), + user_uid: user.uid, type: 'audio' }); // add user's video dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'video'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'video' }); } } else { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - const subscriptions = db.get('subscriptions').value(); - - if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions); // add audio dir to check list dirs_to_check.push({ basePath: audioFolderPath, - dbPath: db.get('files'), type: 'audio' }); // add video dir to check list dirs_to_check.push({ basePath: videoFolderPath, - dbPath: db.get('files'), type: 'video' }); } + if (subscriptions_enabled) { + const subscriptions = await exports.getRecords('subscriptions'); + subscriptions_to_check = subscriptions_to_check.concat(subscriptions); + } + // add subscriptions to check list for (let i = 0; i < subscriptions_to_check.length; i++) { let subscription_to_check = subscriptions_to_check[i]; @@ -176,60 +328,675 @@ function getFileDirectoriesAndDBs() { continue; } dirs_to_check.push({ - basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) + basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name), - dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos') - : db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'), - type: subscription_to_check.type + user_uid: subscription_to_check.user_uid, + type: subscription_to_check.type, + sub_id: subscription_to_check['id'] }); } return dirs_to_check; } -async function importUnregisteredFiles() { - const dirs_to_check = getFileDirectoriesAndDBs(); +exports.importUnregisteredFiles = async () => { + const dirs_to_check = await exports.getFileDirectoriesAndDBs(); // run through check list and check each file to see if it's missing from the db - for (const dir_to_check of dirs_to_check) { + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; // recursively get all files in dir's path const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); - files.forEach(file => { + for (let j = 0; j < files.length; j++) { + const file = files[j]; + // check if file exists in db, if not add it - const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value()) + const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id})) if (!file_is_registered) { // add additional info - registerFileDBManual(dir_to_check.dbPath, file); + await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); logger.verbose(`Added discovered file to the database: ${file.id}`); } + } + } + +} + +exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => { + const preimported_file_paths = []; + + const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + // check if file exists in db, if not add it + const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id}); + if (!file_is_registered) { + // add additional info + await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file); + preimported_file_paths.push(file['path']); + logger.verbose(`Preemptively added subscription file to the database: ${file.id}`); + } + } + return preimported_file_paths; +} + +exports.addMetadataPropertyToDB = async (property_key) => { + try { + const dirs_to_check = await exports.getFileDirectoriesAndDBs(); + const update_obj = {}; + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; + + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); + for (let j = 0; j < files.length; j++) { + const file = files[j]; + if (file[property_key]) { + update_obj[file.uid] = {[property_key]: file[property_key]}; + } + } + } + + return await exports.bulkUpdateRecords('files', 'uid', update_obj); + } catch(err) { + logger.error(err); + return false; + } +} + +exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => { + let new_playlist = { + name: playlist_name, + uids: uids, + id: uuid(), + thumbnailURL: thumbnail_url, + type: type, + registered: Date.now(), + }; + + const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid); + new_playlist.duration = duration; + + new_playlist.user_uid = user_uid ? user_uid : undefined; + + await exports.insertRecordIntoTable('playlists', new_playlist); + + return new_playlist; +} + +exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => { + let playlist = await exports.getRecord('playlists', {id: playlist_id}); + + if (!playlist) { + playlist = await exports.getRecord('categories', {uid: playlist_id}); + if (playlist) { + // category found + const files = await exports.getFiles(user_uid); + utils.addUIDsToCategory(playlist, files); + } + } + + // converts playlists to new UID-based schema + if (playlist && playlist['fileNames'] && !playlist['uids']) { + playlist['uids'] = []; + logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`); + for (let i = 0; i < playlist['fileNames'].length; i++) { + const fileName = playlist['fileNames'][i]; + const uid = await exports.getVideoUIDByID(fileName, user_uid); + if (uid) playlist['uids'].push(uid); + else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); + } + exports.updatePlaylist(playlist, user_uid); + } + + // prevent unauthorized users from accessing the file info + if (require_sharing && !playlist['sharingEnabled']) return null; + + return playlist; +} + +exports.updatePlaylist = async (playlist, user_uid = null) => { + let playlistID = playlist.id; + + const duration = await exports.calculatePlaylistDuration(playlist, user_uid); + playlist.duration = duration; + + return await exports.updateRecord('playlists', {id: playlistID}, playlist); +} + +exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => { + let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj); + + if (!success) { + success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj); + } + + if (!success) { + logger.error(`Could not find playlist or category with ID ${playlist_id}`); + } + + return success; +} + +exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => { + if (!playlist_file_objs) { + playlist_file_objs = []; + for (let i = 0; i < playlist['uids'].length; i++) { + const uid = playlist['uids'][i]; + const file_obj = await exports.getVideo(uid, uuid); + if (file_obj) playlist_file_objs.push(file_obj); + } + } + + return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); +} + +exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => { + const file_obj = await exports.getVideo(uid, uuid); + const type = file_obj.isAudio ? 'audio' : 'video'; + const folderPath = path.dirname(file_obj.path); + const ext = type === 'audio' ? 'mp3' : 'mp4'; + const name = file_obj.id; + const filePathNoExtension = utils.removeFileExtension(file_obj.path); + + var jsonPath = `${file_obj.path}.info.json`; + var altJSONPath = `${filePathNoExtension}.info.json`; + var thumbnailPath = `${filePathNoExtension}.webp`; + var altThumbnailPath = `${filePathNoExtension}.jpg`; + + jsonPath = path.join(__dirname, jsonPath); + altJSONPath = path.join(__dirname, altJSONPath); + + let jsonExists = await fs.pathExists(jsonPath); + let thumbnailExists = await fs.pathExists(thumbnailPath); + + if (!jsonExists) { + if (await fs.pathExists(altJSONPath)) { + jsonExists = true; + jsonPath = altJSONPath; + } + } + + if (!thumbnailExists) { + if (await fs.pathExists(altThumbnailPath)) { + thumbnailExists = true; + thumbnailPath = altThumbnailPath; + } + } + + let fileExists = await fs.pathExists(file_obj.path); + + if (config_api.descriptors[uid]) { + try { + for (let i = 0; i < config_api.descriptors[uid].length; i++) { + config_api.descriptors[uid][i].destroy(); + } + } catch(e) { + + } + } + + let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive) { + const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`); + + // get ID from JSON + + var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); + let id = null; + if (jsonobj) id = jsonobj.id; + + // use subscriptions API to remove video from the archive file, and write it to the blacklist + if (await fs.pathExists(archive_path)) { + const line = id ? await utils.removeIDFromArchive(archive_path, id) : null; + if (blacklistMode && line) await writeToBlacklist(type, line); + } else { + logger.info('Could not find archive file for audio files. Creating...'); + await fs.close(await fs.open(archive_path, 'w')); + } + } + + if (jsonExists) await fs.unlink(jsonPath); + if (thumbnailExists) await fs.unlink(thumbnailPath); + + await exports.removeRecord('files', {uid: uid}); + + if (fileExists) { + await fs.unlink(file_obj.path); + if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) { + return false; + } else { + return true; + } + } else { + // TODO: tell user that the file didn't exist + return true; + } +} + +// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that +exports.getVideoUIDByID = async (file_id, uuid = null) => { + const file_obj = await exports.getRecord('files', {id: file_id}); + return file_obj ? file_obj['uid'] : null; +} + +exports.getVideo = async (file_uid, uuid = null, sub_id = null) => { + return await exports.getRecord('files', {uid: file_uid}); +} + +exports.getFiles = async (uuid = null) => { + return await exports.getRecords('files', {user_uid: uuid}); +} + +exports.setVideoProperty = async (file_uid, assignment_obj) => { + // TODO: check if video exists, throw error if not + await exports.updateRecord('files', {uid: file_uid}, assignment_obj); +} + +// Basic DB functions + +// Create + +exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => { + // local db override + if (using_local_db) { + if (replaceFilter) local_db.get(table).remove(replaceFilter).write(); + local_db.get(table).push(doc).write(); + return true; + } + + if (replaceFilter) await database.collection(table).deleteMany(replaceFilter); + + const output = await database.collection(table).insertOne(doc); + logger.debug(`Inserted doc into ${table}`); + return !!(output['result']['ok']); +} + +exports.insertRecordsIntoTable = async (table, docs, ignore_errors = false) => { + // local db override + if (using_local_db) { + const records_limit = 30000; + if (docs.length < records_limit) { + local_db.get(table).push(...docs).write(); + } else { + for (let i = 0; i < docs.length; i+=records_limit) { + const records_to_push = docs.slice(i, i+records_limit > docs.length ? docs.length : i+records_limit) + local_db.get(table).push(...records_to_push).write(); + } + } + return true; + } + const output = await database.collection(table).insertMany(docs, {ordered: !ignore_errors}); + logger.debug(`Inserted ${output.insertedCount} docs into ${table}`); + return !!(output['result']['ok']); +} + +exports.bulkInsertRecordsIntoTable = async (table, docs) => { + // local db override + if (using_local_db) { + return await exports.insertRecordsIntoTable(table, docs); + } + + // not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed + const table_collection = database.collection(table); + + let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch + + for (let i = 0; i < docs.length; i++) { + bulk.insert(docs[i]); + } + + const output = await bulk.execute(); + return !!(output['result']['ok']); + +} + +// Read + +exports.getRecord = async (table, filter_obj) => { + // local db override + if (using_local_db) { + return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value(); + } + + return await database.collection(table).findOne(filter_obj); +} + +exports.getRecords = async (table, filter_obj = null) => { + // local db override + if (using_local_db) { + return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value(); + } + + return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray(); +} + +// Update + +exports.updateRecord = async (table, filter_obj, update_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write(); + return true; + } + + // sometimes _id will be in the update obj, this breaks mongodb + if (update_obj['_id']) delete update_obj['_id']; + const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj}); + return !!(output['result']['ok']); +} + +exports.updateRecords = async (table, filter_obj, update_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write(); + return true; + } + + const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj}); + return !!(output['result']['ok']); +} + +exports.bulkUpdateRecords = async (table, key_label, update_obj) => { + // local db override + if (using_local_db) { + local_db.get(table).each((record) => { + const item_id_to_update = record[key_label]; + if (!update_obj[item_id_to_update]) return; + + const props_to_update = Object.keys(update_obj[item_id_to_update]); + for (let i = 0; i < props_to_update.length; i++) { + const prop_to_update = props_to_update[i]; + const prop_value = update_obj[item_id_to_update][prop_to_update]; + record[prop_to_update] = prop_value; + } + }).write(); + return true; + } + + const table_collection = database.collection(table); + + let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch + + const item_ids_to_update = Object.keys(update_obj); + + for (let i = 0; i < item_ids_to_update.length; i++) { + const item_id_to_update = item_ids_to_update[i]; + bulk.find({[key_label]: item_id_to_update }).updateOne({ + "$set": update_obj[item_id_to_update] }); } + const output = await bulk.execute(); + return !!(output['result']['ok']); } -async function getVideo(file_uid, uuid, sub_id) { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); - return sub_db_path.find({uid: file_uid}).value(); -} - -async function setVideoProperty(file_uid, assignment_obj, uuid, sub_id) { - const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; - const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); - const file_db_path = sub_db_path.find({uid: file_uid}); - if (!(file_db_path.value())) { - logger.error(`Failed to find file with uid ${file_uid}`); +exports.pushToRecordsArray = async (table, filter_obj, key, value) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write(); + return true; } - sub_db_path.find({uid: file_uid}).assign(assignment_obj).write(); + + const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}}); + return !!(output['result']['ok']); } -module.exports = { - initialize: initialize, - registerFileDB: registerFileDB, - updatePlaylist: updatePlaylist, - getFileDirectoriesAndDBs: getFileDirectoriesAndDBs, - importUnregisteredFiles: importUnregisteredFiles, - getVideo: getVideo, - setVideoProperty: setVideoProperty +exports.pullFromRecordsArray = async (table, filter_obj, key, value) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write(); + return true; + } + + const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}}); + return !!(output['result']['ok']); } + +// Delete + +exports.removeRecord = async (table, filter_obj) => { + // local db override + if (using_local_db) { + applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write(); + return true; + } + + const output = await database.collection(table).deleteOne(filter_obj); + return !!(output['result']['ok']); +} + +exports.removeAllRecords = async (table = null) => { + // local db override + const tables_to_remove = table ? [table] : tables_list; + if (using_local_db) { + logger.debug(`Removing all records from: ${tables_to_remove}`) + for (let i = 0; i < tables_to_remove.length; i++) { + const table_to_remove = tables_to_remove[i]; + local_db.assign({[table_to_remove]: []}).write(); + logger.debug(`Removed all records from ${table_to_remove}`); + } + return true; + } + + let success = true; + logger.debug(`Removing all records from: ${tables_to_remove}`) + for (let i = 0; i < tables_to_remove.length; i++) { + const table_to_remove = tables_to_remove[i]; + + const output = await database.collection(table_to_remove).deleteMany({}); + logger.debug(`Removed all records from ${table_to_remove}`); + success &= !!(output['result']['ok']); + } + return success; +} + +// Stats + +exports.getDBStats = async () => { + const stats_by_table = {}; + for (let i = 0; i < tables_list.length; i++) { + const table = tables_list[i]; + if (table === 'test') continue; + + stats_by_table[table] = await getDBTableStats(table); + } + return {stats_by_table: stats_by_table, using_local_db: using_local_db}; +} + +const getDBTableStats = async (table) => { + const table_stats = {}; + // local db override + if (using_local_db) { + table_stats['records_count'] = local_db.get(table).value().length; + } else { + const stats = await database.collection(table).stats(); + table_stats['records_count'] = stats.count; + } + return table_stats; +} + +// JSON to DB + +exports.generateJSONTables = async (db_json, users_json) => { + // create records + let files = db_json['files'] || []; + let playlists = db_json['playlists'] || []; + let categories = db_json['categories'] || []; + let subscriptions = db_json['subscriptions'] || []; + + const users = users_json['users']; + + for (let i = 0; i < users.length; i++) { + const user = users[i]; + + if (user['files']) { + user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] })); + files = files.concat(user['files']); + } + if (user['playlists']) { + user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] })); + playlists = playlists.concat(user['playlists']); + } + if (user['categories']) { + user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] })); + categories = categories.concat(user['categories']); + } + + if (user['subscriptions']) { + user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] })); + subscriptions = subscriptions.concat(user['subscriptions']); + } + } + + const tables_obj = {}; + + // TODO: use create*Records funcs to strip unnecessary properties + tables_obj.files = createFilesRecords(files, subscriptions); + tables_obj.playlists = playlists; + tables_obj.categories = categories; + tables_obj.subscriptions = createSubscriptionsRecords(subscriptions); + tables_obj.users = createUsersRecords(users); + tables_obj.roles = createRolesRecords(users_json['roles']); + tables_obj.downloads = createDownloadsRecords(db_json['downloads']) + + return tables_obj; +} + +exports.importJSONToDB = async (db_json, users_json) => { + await fs.writeFile(`appdata/db.json.${Date.now()/1000}.bak`, JSON.stringify(db_json, null, 2)); + await fs.writeFile(`appdata/users_db.json.${Date.now()/1000}.bak`, JSON.stringify(users_json, null, 2)); + + await exports.removeAllRecords(); + const tables_obj = await exports.generateJSONTables(db_json, users_json); + + const table_keys = Object.keys(tables_obj); + + let success = true; + for (let i = 0; i < table_keys.length; i++) { + const table_key = table_keys[i]; + if (!tables_obj[table_key] || tables_obj[table_key].length === 0) continue; + success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key], true); + } + + return success; +} + +const createFilesRecords = (files, subscriptions) => { + for (let i = 0; i < subscriptions.length; i++) { + const subscription = subscriptions[i]; + subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined})); + files = files.concat(subscriptions[i]['videos']); + } + + return files; +} + +const createPlaylistsRecords = async (playlists) => { + +} + +const createCategoriesRecords = async (categories) => { + +} + +const createSubscriptionsRecords = (subscriptions) => { + for (let i = 0; i < subscriptions.length; i++) { + delete subscriptions[i]['videos']; + } + + return subscriptions; +} + +const createUsersRecords = (users) => { + users.forEach(user => { + delete user['files']; + delete user['playlists']; + delete user['subscriptions']; + }); + return users; +} + +const createRolesRecords = (roles) => { + const new_roles = []; + Object.keys(roles).forEach(role_key => { + new_roles.push({ + key: role_key, + ...roles[role_key] + }); + }); + return new_roles; +} + +const createDownloadsRecords = (downloads) => { + const new_downloads = []; + Object.keys(downloads).forEach(session_key => { + new_downloads.push({ + key: session_key, + ...downloads[session_key] + }); + }); + return new_downloads; +} + +exports.transferDB = async (local_to_remote) => { + const table_to_records = {}; + for (let i = 0; i < tables_list.length; i++) { + const table = tables_list[i]; + table_to_records[table] = await exports.getRecords(table); + } + + using_local_db = !local_to_remote; + if (local_to_remote) { + // backup local DB + logger.debug('Backup up Local DB...'); + await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`); + const db_connected = await exports.connectToDB(5, true); + if (!db_connected) { + logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.'); + return false; + } + } + success = true; + + logger.debug('Clearing new database before transfer...'); + + await exports.removeAllRecords(); + + logger.debug('Database cleared! Beginning transfer.'); + + for (let i = 0; i < tables_list.length; i++) { + const table = tables_list[i]; + if (!table_to_records[table] || table_to_records[table].length === 0) continue; + success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]); + } + + config_api.setConfigItem('ytdl_use_local_db', using_local_db); + + return success; +} + +/* + This function is necessary to emulate mongodb's ability to search for null or missing values. + A filter of null or undefined for a property will find docs that have that property missing, or have it + null or undefined. We want that same functionality for the local DB as well +*/ +const applyFilterLocalDB = (db_path, filter_obj, operation) => { + const filter_props = Object.keys(filter_obj); + const return_val = db_path[operation](record => { + if (!filter_props) return true; + let filtered = true; + for (let i = 0; i < filter_props.length; i++) { + const filter_prop = filter_props[i]; + const filter_prop_value = filter_obj[filter_prop]; + if (filter_prop_value === undefined || filter_prop_value === null) { + filtered &= record[filter_prop] === undefined || record[filter_prop] === null + } else { + filtered &= record[filter_prop] === filter_prop_value; + } + } + return filtered; + }); + return return_val; +} \ No newline at end of file diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index d32ec93..611bcc8 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu -CMD="node app.js" +CMD="forever app.js" # if the first arg starts with "-" pass it to program if [ "${1#-}" != "$1" ]; then diff --git a/backend/package-lock.json b/backend/package-lock.json index 4067dd9..494042e 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -4,6 +4,19 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==" + }, + "@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "requires": { + "defer-to-connect": "^1.0.1" + } + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -87,6 +100,11 @@ "@types/mime": "*" } }, + "@ungap/promise-all-settled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", + "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -113,24 +131,56 @@ } }, "ansi-align": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", - "integrity": "sha1-w2rsy6VjuJzrVW82kPCx2eNUf38=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", + "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", "requires": { - "string-width": "^2.0.0" + "string-width": "^3.0.0" + }, + "dependencies": { + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } } }, + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==" + }, "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" }, "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "requires": { - "color-convert": "^1.9.0" + "color-convert": "^2.0.1" + }, + "dependencies": { + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + } } }, "any-promise": { @@ -139,9 +189,9 @@ "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" }, "anymatch": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", - "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -209,6 +259,11 @@ } } }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -306,14 +361,14 @@ } }, "binary-extensions": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", - "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, "bl": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", - "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -350,17 +405,18 @@ } }, "boxen": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", - "integrity": "sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", "requires": { - "ansi-align": "^2.0.0", - "camelcase": "^4.0.0", - "chalk": "^2.0.1", - "cli-boxes": "^1.0.0", - "string-width": "^2.0.0", - "term-size": "^1.2.0", - "widest-line": "^2.0.0" + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" } }, "brace-expansion": { @@ -380,6 +436,16 @@ "fill-range": "^7.0.1" } }, + "browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" + }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" + }, "buffer": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz", @@ -462,15 +528,31 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, - "camelcase": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", - "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" + "cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "dependencies": { + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" + } + } }, - "capture-stack-trace": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", - "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==" + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, "caseless": { "version": "0.12.0", @@ -486,13 +568,27 @@ } }, "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + } } }, "charenc": { @@ -501,29 +597,62 @@ "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, "chokidar": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.3.1.tgz", - "integrity": "sha512-4QYCEWOcK3OJrxwvyyAOxFuhpvOVCYkr33LPfFNBjAD/w3sEzWsp2BUOkI4l9bHvWioAd0rc6NlHUOEaWkTeqg==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", - "fsevents": "~2.1.2", + "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.3.0" + "readdirp": "~3.5.0" } }, "ci-info": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", - "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==" }, "cli-boxes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", - "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=" + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==" + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "requires": { + "mimic-response": "^1.0.0" + } }, "color": { "version": "3.0.0", @@ -673,16 +802,16 @@ } }, "configstore": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz", - "integrity": "sha512-vtv5HtGjcYUgFrXc6Kx747B83MRRVS5R1VTEQoXvuP+kMI+if6uywV0nDGoiydJRy4yk7h9od5Og0kxx4zUXmw==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", "requires": { - "dot-prop": "^4.1.0", + "dot-prop": "^5.2.0", "graceful-fs": "^4.1.2", - "make-dir": "^1.0.0", - "unique-string": "^1.0.0", - "write-file-atomic": "^2.0.0", - "xdg-basedir": "^3.0.0" + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" } }, "connected-domain": { @@ -735,14 +864,6 @@ "readable-stream": "^3.4.0" } }, - "create-error-class": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", - "integrity": "sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=", - "requires": { - "capture-stack-trace": "^1.0.0" - } - }, "cross-spawn": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", @@ -759,9 +880,9 @@ "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" }, "crypto-random-string": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", - "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==" }, "dashdash": { "version": "1.14.1", @@ -779,16 +900,39 @@ "ms": "2.0.0" } }, + "decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==" + }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "requires": { + "mimic-response": "^1.0.0" + } + }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, + "defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -841,12 +985,17 @@ } } }, + "diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==" + }, "dot-prop": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", - "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", "requires": { - "is-obj": "^1.0.0" + "is-obj": "^2.0.0" } }, "dtrace-provider": { @@ -909,6 +1058,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, "enabled": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", @@ -935,15 +1089,25 @@ "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", "integrity": "sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==" }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" }, "etag": { "version": "1.8.1", @@ -1061,6 +1225,20 @@ "unpipe": "~1.0.0" } }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, + "flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==" + }, "fluent-ffmpeg": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", @@ -1132,9 +1310,9 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", - "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, "fstream": { @@ -1148,6 +1326,11 @@ "rimraf": "2" } }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, "get-stream": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", @@ -1178,48 +1361,46 @@ } }, "glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "requires": { "is-glob": "^4.0.1" } }, "global-dirs": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz", - "integrity": "sha1-sxnA3UYH81PzvpzKTHL8FIxJ9EU=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.1.0.tgz", + "integrity": "sha512-MG6kdOUh/xBnyo9cJFeIKkLEc1AyFq42QTU4XiX51i2NEdxLxLWXIjEjmqKeSuKR7pAZjTqUVoT2b2huxVLgYQ==", "requires": { - "ini": "^1.3.4" + "ini": "1.3.7" } }, "got": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", - "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", "requires": { - "create-error-class": "^3.0.0", + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "is-redirect": "^1.0.0", - "is-retry-allowed": "^1.0.0", - "is-stream": "^1.0.0", - "lowercase-keys": "^1.0.0", - "safe-buffer": "^5.0.1", - "timed-out": "^4.0.0", - "unzip-response": "^2.0.1", - "url-parse-lax": "^1.0.0" + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" }, "dependencies": { "get-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" - }, - "is-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } } } }, @@ -1228,6 +1409,11 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==" + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -1247,6 +1433,11 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, + "has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==" + }, "hashish": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", @@ -1255,6 +1446,11 @@ "traverse": ">=0.2.4" } }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + }, "hh-mm-ss": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz", @@ -1263,6 +1459,11 @@ "zero-fill": "^2.2.3" } }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1340,9 +1541,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==" }, "ipaddr.js": { "version": "1.9.1", @@ -1368,11 +1569,11 @@ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-ci": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", - "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", "requires": { - "ci-info": "^1.5.0" + "ci-info": "^2.0.0" } }, "is-extglob": { @@ -1394,18 +1595,18 @@ } }, "is-installed-globally": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.1.0.tgz", - "integrity": "sha1-Df2Y9akRFxbdU13aZJL2e/PSWoA=", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", "requires": { - "global-dirs": "^0.1.0", - "is-path-inside": "^1.0.0" + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" } }, "is-npm": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", - "integrity": "sha1-8vtjpl5JBbQGyGBydloaTceTufQ=" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==" }, "is-number": { "version": "7.0.0", @@ -1413,33 +1614,25 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "is-obj": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", - "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, "is-path-inside": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", - "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", - "requires": { - "path-is-inside": "^1.0.1" - } + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==" + }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==" }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" }, - "is-redirect": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", - "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=" - }, - "is-retry-allowed": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", - "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==" - }, "is-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", @@ -1450,6 +1643,11 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -1465,11 +1663,24 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "requires": { + "argparse": "^2.0.1" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" + }, "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", @@ -1556,6 +1767,14 @@ "safe-buffer": "^5.0.1" } }, + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "requires": { + "json-buffer": "3.0.0" + } + }, "kuler": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz", @@ -1565,11 +1784,11 @@ } }, "latest-version": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", - "integrity": "sha1-ogU4P+oyKzO1rjsYq+4NwvNW7hU=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", "requires": { - "package-json": "^4.0.0" + "package-json": "^6.3.0" } }, "lazystream": { @@ -1667,10 +1886,18 @@ "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "requires": { + "p-locate": "^5.0.0" + } + }, "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.defaults": { "version": "4.2.0", @@ -1727,6 +1954,38 @@ "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" }, + "log-symbols": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "requires": { + "chalk": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "logform": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz", @@ -1763,21 +2022,19 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" }, - "lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, "make-dir": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", - "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", "requires": { - "pify": "^3.0.0" + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } } }, "md5": { @@ -1795,6 +2052,12 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -1841,6 +2104,11 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1862,11 +2130,126 @@ "minimist": "^1.2.5" } }, + "mocha": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", + "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "requires": { + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", + "minimatch": "3.0.4", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", + "wide-align": "1.1.3", + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==" + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" + }, + "supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "moment": { "version": "2.29.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "mongodb": { + "version": "3.6.9", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.9.tgz", + "integrity": "sha512-1nSCKgSunzn/CXwgOWgbPHUWOO5OfERcuOWISmqd610jn0s8BU9K4879iJVabqgpPPbA6hO7rG48eq+fGED3Mg==", + "requires": { + "bl": "^2.2.1", + "bson": "^1.1.4", + "denque": "^1.4.1", + "optional-require": "^1.0.3", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + }, + "dependencies": { + "bl": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", + "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + } + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -1993,9 +2376,9 @@ } }, "nodemon": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.2.tgz", - "integrity": "sha512-GWhYPMfde2+M0FsHnggIHXTqPDHXia32HRhh6H0d75Mt9FKUoCBvumNHr7LdrpPBTKxsWmIEOjoN+P4IU6Hcaw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", + "integrity": "sha512-XHzK69Awgnec9UzHr1kc8EomQh4sjTQ8oRf8TsGrSmHDx9/UmiGG9E/mM3BuTfNeFwdNBvrqQq/RHL0xIeyFOA==", "requires": { "chokidar": "^3.2.2", "debug": "^3.2.6", @@ -2005,22 +2388,22 @@ "semver": "^5.7.1", "supports-color": "^5.5.0", "touch": "^3.1.0", - "undefsafe": "^2.0.2", - "update-notifier": "^2.5.0" + "undefsafe": "^2.0.3", + "update-notifier": "^4.1.0" }, "dependencies": { "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "requires": { "ms": "^2.1.1" } }, "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" } } }, @@ -2037,6 +2420,11 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, + "normalize-url": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==" + }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -2089,20 +2477,53 @@ "mimic-fn": "^2.1.0" } }, + "optional-require": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.3.tgz", + "integrity": "sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA==" + }, + "p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==" + }, "p-finally": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz", "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==" }, - "package-json": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/package-json/-/package-json-4.0.1.tgz", - "integrity": "sha1-iGmgQBJTZhxMTKPabCEh7VVfXu0=", + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "requires": { - "got": "^6.7.1", - "registry-auth-token": "^3.0.1", - "registry-url": "^3.0.3", - "semver": "^5.1.0" + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "requires": { + "p-limit": "^3.0.2" + } + }, + "package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "requires": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } } }, "parseurl": { @@ -2184,16 +2605,16 @@ "resolved": "https://registry.npmjs.org/passport-strategy/-/passport-strategy-1.0.0.tgz", "integrity": "sha1-tVOaqPwiWj0a0XlHbd8ja0QPUuQ=" }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" - }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -2215,9 +2636,9 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "picomatch": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", - "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", + "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==" }, "pify": { "version": "3.0.0", @@ -2230,9 +2651,9 @@ "integrity": "sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw=" }, "prepend-http": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", - "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" }, "process-nextick-args": { "version": "2.0.1", @@ -2261,20 +2682,15 @@ "table-parser": "^0.1.3" } }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "pstree.remy": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.7.tgz", - "integrity": "sha512-xsMgrUwRpuGskEzBFkH8NmTimbZ5PcPup0LA8JJkHIm2IMUbQcpo3yeLNWVrufEYjh8YwtSVh0xz6UeWc5Oh5A==" + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==" }, "pump": { "version": "3.0.0", @@ -2290,11 +2706,27 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, + "pupa": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.1.1.tgz", + "integrity": "sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A==", + "requires": { + "escape-goat": "^2.0.0" + } + }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -2348,28 +2780,27 @@ } }, "readdirp": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.3.0.tgz", - "integrity": "sha512-zz0pAkSPOXXm1viEwygWIPSPkcBYjW1xU5j/JBh5t9bGCJwa6f9+BJa6VaB2g+b55yVrmXzqkyLf4xaWYM0IkQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", "requires": { - "picomatch": "^2.0.7" + "picomatch": "^2.2.1" } }, "registry-auth-token": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", - "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.1.tgz", + "integrity": "sha512-6gkSb4U6aWJB4SF2ZvLb76yCBjcvufXBqvvEx1HbmKPkutswjW1xNVRY0+daljIYRbogN7O0etYSlbiaEQyMyw==", "requires": { - "rc": "^1.1.6", - "safe-buffer": "^5.0.1" + "rc": "^1.2.8" } }, "registry-url": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", - "integrity": "sha1-PU74cPc93h138M+aOBQyRE4XSUI=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", "requires": { - "rc": "^1.0.1" + "rc": "^1.2.8" } }, "request": { @@ -2411,6 +2842,19 @@ } } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "requires": { + "lowercase-keys": "^1.0.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -2435,17 +2879,33 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, "semver-diff": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz", - "integrity": "sha1-S7uEN8jTfksM8aaP1ybsbWRdbTY=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", "requires": { - "semver": "^5.0.3" + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } } }, "send": { @@ -2475,6 +2935,14 @@ } } }, + "serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "requires": { + "randombytes": "^2.1.0" + } + }, "serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", @@ -2530,6 +2998,15 @@ "is-arrayish": "^0.3.1" } }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2578,12 +3055,38 @@ "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" }, "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "requires": { + "ansi-regex": "^5.0.0" + } + } } }, "string_decoder": { @@ -2595,18 +3098,13 @@ } }, "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "requires": { - "ansi-regex": "^3.0.0" + "ansi-regex": "^4.1.0" } }, - "strip-eof": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" - }, "strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", @@ -2646,87 +3144,9 @@ } }, "term-size": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", - "integrity": "sha1-RYuDiH8oj8Vtb/+/rSYuJmOO+mk=", - "requires": { - "execa": "^0.7.0" - }, - "dependencies": { - "cross-spawn": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", - "requires": { - "lru-cache": "^4.0.1", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - } - }, - "execa": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", - "requires": { - "cross-spawn": "^5.0.1", - "get-stream": "^3.0.0", - "is-stream": "^1.1.0", - "npm-run-path": "^2.0.0", - "p-finally": "^1.0.0", - "signal-exit": "^3.0.0", - "strip-eof": "^1.0.0" - } - }, - "get-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" - }, - "is-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" - }, - "npm-run-path": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", - "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", - "requires": { - "path-key": "^2.0.0" - } - }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" - }, - "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" - }, - "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "requires": { - "shebang-regex": "^1.0.0" - } - }, - "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" - }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "requires": { - "isexe": "^2.0.0" - } - } - } + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz", + "integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==" }, "text-hex": { "version": "1.0.0", @@ -2749,10 +3169,10 @@ "thenify": ">= 3.1.0 < 4" } }, - "timed-out": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", - "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" + "to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==" }, "to-regex-range": { "version": "5.0.1", @@ -2807,6 +3227,11 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==" + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2821,6 +3246,14 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "requires": { + "is-typedarray": "^1.0.0" + } + }, "undefsafe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", @@ -2830,11 +3263,11 @@ } }, "unique-string": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", - "integrity": "sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", "requires": { - "crypto-random-string": "^1.0.0" + "crypto-random-string": "^2.0.0" } }, "universalify": { @@ -2847,11 +3280,6 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "unzip-response": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-2.0.1.tgz", - "integrity": "sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=" - }, "unzipper": { "version": "0.10.10", "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.10.tgz", @@ -2886,20 +3314,23 @@ } }, "update-notifier": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-2.5.0.tgz", - "integrity": "sha512-gwMdhgJHGuj/+wHJJs9e6PcCszpxR1b236igrOkUofGhqJuG+amlIKwApH1IW1WWl7ovZxsX49lMBWLxSdm5Dw==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", + "integrity": "sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A==", "requires": { - "boxen": "^1.2.1", - "chalk": "^2.0.1", - "configstore": "^3.0.0", + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", "import-lazy": "^2.1.0", - "is-ci": "^1.0.10", - "is-installed-globally": "^0.1.0", - "is-npm": "^1.0.0", - "latest-version": "^3.0.0", - "semver-diff": "^2.0.0", - "xdg-basedir": "^3.0.0" + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "pupa": "^2.0.1", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" } }, "uri-js": { @@ -2911,11 +3342,11 @@ } }, "url-parse-lax": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", - "integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", "requires": { - "prepend-http": "^1.0.1" + "prepend-http": "^2.0.0" } }, "util-deprecate": { @@ -2987,12 +3418,44 @@ "isexe": "^2.0.0" } }, - "widest-line": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", - "integrity": "sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==", + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "requires": { - "string-width": "^2.1.1" + "string-width": "^1.0.2 || 2" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "requires": { + "string-width": "^4.0.0" } }, "winston": { @@ -3051,35 +3514,108 @@ } } }, + "workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==" + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write-file-atomic": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", - "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", "requires": { - "graceful-fs": "^4.1.11", "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.2" + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" } }, "xdg-basedir": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", - "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==" }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } + }, + "yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==" + }, + "yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "requires": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + }, + "dependencies": { + "camelcase": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==" + } + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" }, "youtube-dl": { "version": "3.0.2", diff --git a/backend/package.json b/backend/package.json index 862a09c..b2cd7c1 100644 --- a/backend/package.json +++ b/backend/package.json @@ -5,7 +5,8 @@ "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "nodemon -q app.js" + "start": "nodemon app.js", + "debug": "set YTDL_MODE=debug && node app.js" }, "nodemonConfig": { "ignore": [ @@ -14,7 +15,8 @@ "public/*" ], "watch": [ - "restart.json" + "restart_update.json", + "restart_general.json" ] }, "repository": { @@ -43,11 +45,13 @@ "lowdb": "^1.0.0", "md5": "^2.2.1", "merge-files": "^0.1.2", + "mocha": "^8.4.0", "moment": "^2.29.1", + "mongodb": "^3.6.9", "multer": "^1.4.2", "node-fetch": "^2.6.1", "node-id3": "^0.1.14", - "nodemon": "^2.0.2", + "nodemon": "^2.0.7", "passport": "^0.4.1", "passport-http": "^0.3.0", "passport-jwt": "^4.0.0", diff --git a/backend/subscriptions.js b/backend/subscriptions.js index b3e2f1e..cd15f40 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -14,13 +14,13 @@ const debugMode = process.env.YTDL_MODE === 'debug'; var logger = null; var db = null; var users_db = null; -var db_api = null; +let db_api = null; -function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } +function setDB(input_db_api) { db_api = input_db_api } function setLogger(input_logger) { logger = input_logger; } -function initialize(input_db, input_users_db, input_logger, input_db_api) { - setDB(input_db, input_users_db, input_db_api); +function initialize(input_db_api, input_logger) { + setDB(input_db_api); setLogger(input_logger); } @@ -34,12 +34,7 @@ async function subscribe(sub, user_uid = null) { sub.isPlaylist = sub.url.includes('playlist'); sub.videos = []; - let url_exists = false; - - if (user_uid) - url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value() - else - url_exists = !!db.get('subscriptions').find({url: sub.url}).value(); + let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid})); if (!sub.name && url_exists) { logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`); @@ -48,19 +43,12 @@ async function subscribe(sub, user_uid = null) { return; } - // add sub to db - let sub_db = null; - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write(); - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - } else { - db.get('subscriptions').push(sub).write(); - sub_db = db.get('subscriptions').find({id: sub.id}); - } + sub['user_uid'] = user_uid ? user_uid : undefined; + await db_api.insertRecordIntoTable('subscriptions', sub); + let success = await getSubscriptionInfo(sub, user_uid); if (success) { - sub = sub_db.value(); getVideosForSub(sub, user_uid); } else { logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') @@ -91,8 +79,8 @@ async function getSubscriptionInfo(sub, user_uid = null) { } } - return new Promise(resolve => { - youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { + return new Promise(async resolve => { + youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => { if (debugMode) { logger.info('Subscribe: got info for subscription ' + sub.id); } @@ -122,10 +110,7 @@ async function getSubscriptionInfo(sub, user_uid = null) { } // if it's now valid, update if (sub.name) { - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); - else - db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); + await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name}); } } @@ -141,10 +126,8 @@ async function getSubscriptionInfo(sub, user_uid = null) { // updates subscription sub.archive = archive_dir; - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); - else - db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); + + await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir}); } // TODO: get even more info @@ -166,10 +149,8 @@ async function unsubscribe(sub, deleteMode, user_uid = null) { let result_obj = { success: false, error: '' }; let id = sub.id; - if (user_uid) - users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write(); - else - db.get('subscriptions').remove({id: id}).write(); + await db_api.removeRecord('subscriptions', {id: id}); + await db_api.removeAllRecords('files', {sub_id: id}); // failed subs have no name, on unsubscribe they shouldn't error if (!sub.name) { @@ -191,20 +172,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) { } async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { + // TODO: combine this with deletefile let basePath = null; - let sub_db = null; - if (user_uid) { - basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - } else { - basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - sub_db = db.get('subscriptions').find({id: sub.id}); - } + basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions') + : config_api.getConfigItem('ytdl_subscriptions_base_path'); const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const appendedBasePath = getAppendedBasePath(sub, basePath); const name = file; let retrievedID = null; - sub_db.get('videos').remove({uid: file_uid}).write(); + + await db_api.removeRecord('files', {uid: file_uid}); let filePath = appendedBasePath; const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' @@ -243,7 +220,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, const archive_path = path.join(sub.archive, 'archive.txt') // if archive exists, remove line with video ID if (await fs.pathExists(archive_path)) { - await removeIDFromArchive(archive_path, retrievedID); + utils.removeIDFromArchive(archive_path, retrievedID); } } return true; @@ -255,14 +232,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, } async function getVideosForSub(sub, user_uid = null) { - // get sub_db - let sub_db = null; - if (user_uid) - sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); - else - sub_db = db.get('subscriptions').find({id: sub.id}); - - const latest_sub_obj = sub_db.value(); + const latest_sub_obj = await getSubscription(sub.id); if (!latest_sub_obj || latest_sub_obj['downloading']) { return false; } @@ -277,6 +247,7 @@ async function getVideosForSub(sub, user_uid = null) { basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); let appendedBasePath = getAppendedBasePath(sub, basePath); + fs.ensureDirSync(appendedBasePath); let multiUserMode = null; if (user_uid) { @@ -291,9 +262,18 @@ async function getVideosForSub(sub, user_uid = null) { // get videos logger.verbose('Subscription: getting videos for subscription ' + sub.name); - return new Promise(resolve => { - youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) { + return new Promise(async resolve => { + const preimported_file_paths = []; + const PREIMPORT_INTERVAL = 5000; + const preregister_check = setInterval(async () => { + if (sub.streamingOnly) return; + await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath); + }, PREIMPORT_INTERVAL); + youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { + // cleanup updateSubscriptionProperty(sub, {downloading: false}, user_uid); + clearInterval(preregister_check); + logger.verbose('Subscription: finished check for ' + sub.name); if (err && !output) { logger.error(err.stderr ? err.stderr : err.message); @@ -303,7 +283,7 @@ async function getVideosForSub(sub, user_uid = null) { const outputs = err.stdout.split(/\r\n|\r|\n/); for (let i = 0; i < outputs.length; i++) { const output = JSON.parse(outputs[i]); - handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) + await handleOutputJSON(sub, output, i === 0, multiUserMode) if (err.stderr.includes(output['id']) && archive_path) { // we found a video that errored! add it to the archive to prevent future errors if (sub.archive) { @@ -337,7 +317,7 @@ async function getVideosForSub(sub, user_uid = null) { } const reset_videos = i === 0; - handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); + await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos); } if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { @@ -351,6 +331,7 @@ async function getVideosForSub(sub, user_uid = null) { }, err => { logger.error(err); updateSubscriptionProperty(sub, {downloading: false}, user_uid); + clearInterval(preregister_check); }); } @@ -433,8 +414,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de return downloadConfig; } -function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { - if (sub.streamingOnly) { +async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) { + // TODO: remove streaming only mode + if (false && sub.streamingOnly) { if (reset_videos) { sub_db.assign({videos: []}).write(); } @@ -448,12 +430,15 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ path_object = path.parse(output_json['_filename']); const path_string = path.format(path_object); - if (sub_db.get('videos').find({path: path_string}).value()) { + const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id}); + if (file_exists) { + // TODO: fix issue where files of different paths due to custom path get downloaded multiple times // file already exists in DB, return early to avoid reseting the download date return; } - db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); + await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id); + const url = output_json['webpage_url']; if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) { @@ -466,73 +451,41 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ } } -function getSubscriptions(user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').value(); - else - return db.get('subscriptions').value(); +async function getSubscriptions(user_uid = null) { + return await db_api.getRecords('subscriptions', {user_uid: user_uid}); } -function getAllSubscriptions() { - let subscriptions = null; +async function getAllSubscriptions() { + const all_subs = await db_api.getRecords('subscriptions'); const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode) { - subscriptions = []; - let users = users_db.get('users').value(); - for (let i = 0; i < users.length; i++) { - if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); - } - } else { - subscriptions = getSubscriptions(); - } - return subscriptions; + return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode); } -function getSubscription(subID, user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); - else - return db.get('subscriptions').find({id: subID}).value(); +async function getSubscription(subID) { + return await db_api.getRecord('subscriptions', {id: subID}); } -function getSubscriptionByName(subName, user_uid = null) { - if (user_uid) - return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value(); - else - return db.get('subscriptions').find({name: subName}).value(); +async function getSubscriptionByName(subName, user_uid = null) { + return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid}); } -function updateSubscription(sub, user_uid = null) { - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write(); - } else { - db.get('subscriptions').find({id: sub.id}).assign(sub).write(); - } +async function updateSubscription(sub, user_uid = null) { + await db_api.updateRecord('subscriptions', {id: sub.id}, sub); return true; } -function updateSubscriptionPropertyMultiple(subs, assignment_obj) { - subs.forEach(sub => { - updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); +async function updateSubscriptionPropertyMultiple(subs, assignment_obj) { + subs.forEach(async sub => { + await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); }); } -function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { - if (user_uid) { - users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); - } else { - db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); - } +async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { + // TODO: combine with updateSubscription + await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj); return true; } -function subExists(subID, user_uid = null) { - if (user_uid) - return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); - else - return !!db.get('subscriptions').find({id: subID}).value(); -} - async function setFreshUploads(sub, user_uid) { const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); sub.videos.forEach(async video => { @@ -548,7 +501,7 @@ async function checkVideosForFreshUploads(sub, user_uid) { const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); sub.videos.forEach(async video => { if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) { - checkVideoIfBetterExists(video, sub, user_uid) + await checkVideoIfBetterExists(video, sub, user_uid) } }); } @@ -558,14 +511,14 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path); logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); // simulate a download to verify that a better version exists - youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => { + youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => { if (err) { // video is not available anymore for whatever reason } else if (output) { const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; if (output[metric_to_compare] > file_obj[metric_to_compare]) { // download new video as the simulated one is better - youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => { + youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => { if (err) { logger.verbose(`Failed to download better version of video ${file_obj['id']}`); } else if (output) { @@ -586,33 +539,6 @@ function getAppendedBasePath(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } -async function removeIDFromArchive(archive_path, id) { - let data = await fs.readFile(archive_path, {encoding: 'utf-8'}); - if (!data) { - logger.error('Archive could not be found.'); - return; - } - - let dataArray = data.split('\n'); // convert file data in an array - const searchKeyword = id; // we are looking for a line, contains, key word id in the file - let lastIndex = -1; // let say, we have not found the keyword - - for (let index=0; index { + return `${timestamp} ${level.toUpperCase()}: ${message}`; +}); + +let debugMode = process.env.YTDL_MODE === 'debug'; + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine(winston.format.timestamp(), defaultFormat), + defaultMeta: {}, + transports: [ + // + // - Write to all logs with level `info` and below to `combined.log` + // - Write all logs error (and below) to `error.log`. + // + new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), + new winston.transports.File({ filename: 'appdata/logs/combined.log' }), + new winston.transports.Console({level: 'debug', name: 'console'}) + ] +}); + +var auth_api = require('../authentication/auth'); +var db_api = require('../db'); +const utils = require('../utils'); +const subscriptions_api = require('../subscriptions'); +const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); + +db_api.initialize(db, users_db, logger); + + +describe('Database', async function() { + describe('Import', async function() { + it('Migrate', async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords(); + const success = await db_api.importJSONToDB(db.value(), users_db.value()); + assert(success); + }); + + it('Transfer to remote', async function() { + await db_api.removeAllRecords('test'); + await db_api.insertRecordIntoTable('test', {test: 'test'}); + + await db_api.transferDB(true); + const success = await db_api.getRecord('test', {test: 'test'}); + assert(success); + }); + + it('Transfer to local', async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('test'); + await db_api.insertRecordIntoTable('test', {test: 'test'}); + + await db_api.transferDB(false); + const success = await db_api.getRecord('test', {test: 'test'}); + assert(success); + }); + }); + + describe('Export', function() { + + }); + + + describe('Basic functions', async function() { + beforeEach(async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('test'); + }); + it('Add and read record', async function() { + await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); + const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); + assert(added_record['test_add'] === 'test'); + await db_api.removeRecord('test', {test_add: 'test'}); + }); + + it('Update record', async function() { + await db_api.insertRecordIntoTable('test', {test_update: 'test'}); + await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); + const updated_record = await db_api.getRecord('test', {test_update: 'test'}); + assert(updated_record['added_field']); + await db_api.removeRecord('test', {test_update: 'test'}); + }); + + it('Remove record', async function() { + await db_api.insertRecordIntoTable('test', {test_remove: 'test'}); + const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'}); + assert(delete_succeeded); + const deleted_record = await db_api.getRecord('test', {test_remove: 'test'}); + assert(!deleted_record); + }); + + it('Push to record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []}); + await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 1); + }); + + it('Pull from record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']}); + await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 0); + }); + + it('Bulk add', async function() { + const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 + const test_records = []; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + test_records.push({ + uid: uuid() + }); + } + const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records); + + const received_records = await db_api.getRecords('test'); + assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); + }); + + it('Bulk update', async function() { + // bulk add records + const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 + const test_records = []; + const update_obj = {}; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + const test_uid = uuid(); + test_records.push({ + uid: test_uid + }); + update_obj[test_uid] = {added_field: true}; + } + let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); + assert(success); + + // makes sure they are added + const received_records = await db_api.getRecords('test'); + assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); + + success = await db_api.bulkUpdateRecords('test', 'uid', update_obj); + assert(success); + + const received_updated_records = await db_api.getRecords('test'); + for (let i = 0; i < received_updated_records.length; i++) { + success &= received_updated_records[i]['added_field']; + } + assert(success); + }); + + it('Stats', async function() { + const stats = await db_api.getDBStats(); + assert(stats); + }); + + it('Query speed', async function() { + this.timeout(120000); + const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000 + const test_records = []; + let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943'; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + const uid = uuid(); + if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid; + test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon: http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632}); + } + const insert_start = Date.now(); + let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); + const insert_end = Date.now(); + + console.log(`Insert time: ${(insert_end - insert_start)/1000}s`); + + const query_start = Date.now(); + const random_record = await db_api.getRecord('test', {uid: random_uid}); + const query_end = Date.now(); + + console.log(random_record) + + console.log(`Query time: ${(query_end - query_start)/1000}s`); + + success = !!random_record; + + assert(success); + }); + }); +}); + +describe('Multi User', async function() { + let user = null; + const user_to_test = 'admin'; + const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; + const playlist_to_test = 'ysabVZz4x'; + beforeEach(async function() { + await db_api.connectToDB(); + auth_api.initialize(db_api, logger); + subscriptions_api.initialize(db_api, logger); + user = await auth_api.login('admin', 'pass'); + }); + describe('Authentication', function() { + it('login', async function() { + assert(user); + }); + }); + describe('Video player - normal', function() { + const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e'; + it('Get video', async function() { + const video_obj = db_api.getVideo(video_to_test, 'admin'); + assert(video_obj); + }); + + it('Video access - disallowed', async function() { + await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test); + const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + assert(!video_obj); + }); + + it('Video access - allowed', async function() { + await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test); + const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + assert(video_obj); + }); + }); + describe('Zip generators', function() { + it('Playlist zip generator', async function() { + const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test); + assert(playlist); + const playlist_files_to_download = []; + for (let i = 0; i < playlist['uids'].length; i++) { + const uid = playlist['uids'][i]; + const playlist_file = await db_api.getVideo(uid, user_to_test); + playlist_files_to_download.push(playlist_file); + } + const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download); + const zip_exists = fs.pathExistsSync(zip_path); + assert(zip_exists); + if (zip_exists) fs.unlinkSync(zip_path); + }); + + it('Subscription zip generator', async function() { + const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test); + const sub_videos = await db_api.getRecords('files', {sub_id: sub.id}); + assert(sub); + const sub_files_to_download = []; + for (let i = 0; i < sub_videos.length; i++) { + const sub_file = sub_videos[i]; + sub_files_to_download.push(sub_file); + } + const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download); + const zip_exists = fs.pathExistsSync(zip_path); + assert(zip_exists); + if (zip_exists) fs.unlinkSync(zip_path); + }); + }); + // describe('Video player - subscription', function() { + // const sub_to_test = ''; + // const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e'; + // it('Get video', async function() { + // const video_obj = db_api.getVideo(video_to_test, 'admin', ); + // assert(video_obj); + // }); + + // it('Video access - disallowed', async function() { + // await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test); + // const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + // assert(!video_obj); + // }); + + // it('Video access - allowed', async function() { + // await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test); + // const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + // assert(video_obj); + // }); + // }); + +}); \ No newline at end of file diff --git a/backend/utils.js b/backend/utils.js index cd7c23d..dc425e8 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -1,6 +1,7 @@ -var fs = require('fs-extra') -var path = require('path') +const fs = require('fs-extra') +const path = require('path') const config_api = require('./config'); +const archiver = require('archiver'); const is_windows = process.platform === 'win32'; @@ -52,6 +53,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) { return files; } +async function createContainerZipFile(container_obj, container_file_objs) { + const container_files_to_download = []; + for (let i = 0; i < container_file_objs.length; i++) { + const container_file_obj = container_file_objs[i]; + container_files_to_download.push(container_file_obj.path); + } + return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download); +} + +async function createZipFile(zip_file_path, file_paths) { + let output = fs.createWriteStream(zip_file_path); + + var archive = archiver('zip', { + gzip: true, + zlib: { level: 9 } // Sets the compression level. + }); + + archive.on('error', function(err) { + logger.error(err); + throw err; + }); + + // pipe archive data to the output file + archive.pipe(output); + + for (let file_path of file_paths) { + const file_name = path.parse(file_path).base; + archive.file(file_path, {name: file_name}) + } + + await archive.finalize(); + + // wait a tiny bit for the zip to reload in fs + await wait(100); + return zip_file_path; +} + function getJSONMp4(name, customPath, openReadPerms = false) { var obj = null; // output if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path'); @@ -84,6 +122,21 @@ function getJSONMp3(name, customPath, openReadPerms = false) { return obj; } +function getJSON(file_path, type) { + const ext = type === 'audio' ? '.mp3' : '.mp4'; + let obj = null; + var jsonPath = removeFileExtension(file_path) + '.info.json'; + var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`; + if (fs.existsSync(jsonPath)) + { + obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + } else if (fs.existsSync(alternateJsonPath)) { + obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); + } + else obj = 0; + return obj; +} + function getJSONByType(type, name, customPath, openReadPerms = false) { return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) } @@ -105,6 +158,23 @@ function getDownloadedThumbnail(name, type, customPath = null) { return null; } +function getDownloadedThumbnail2(file_path, type) { + const file_path_no_extension = removeFileExtension(file_path); + + let jpgPath = file_path_no_extension + '.jpg'; + let webpPath = file_path_no_extension + '.webp'; + let pngPath = file_path_no_extension + '.png'; + + if (fs.existsSync(jpgPath)) + return jpgPath; + else if (fs.existsSync(webpPath)) + return webpPath; + else if (fs.existsSync(pngPath)) + return pngPath; + else + return null; +} + function getExpectedFileSize(input_info_jsons) { // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons]; @@ -152,6 +222,28 @@ function fixVideoMetadataPerms(name, type, customPath = null) { } } +function fixVideoMetadataPerms2(file_path, type) { + if (is_windows) return; + + const ext = type === 'audio' ? '.mp3' : '.mp4'; + + const file_path_no_extension = removeFileExtension(file_path); + + const files_to_fix = [ + // JSONs + file_path_no_extension + '.info.json', + file_path_no_extension + ext + '.info.json', + // Thumbnails + file_path_no_extension + '.webp', + file_path_no_extension + '.jpg' + ]; + + for (const file of files_to_fix) { + if (!fs.existsSync(file)) continue; + fs.chmodSync(file, 0o644); + } +} + function deleteJSONFile(name, type, customPath = null) { if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path'); @@ -164,6 +256,64 @@ function deleteJSONFile(name, type, customPath = null) { if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); } +function deleteJSONFile2(file_path, type) { + const ext = type === 'audio' ? '.mp3' : '.mp4'; + + const file_path_no_extension = removeFileExtension(file_path); + + let json_path = file_path_no_extension + '.info.json'; + let alternate_json_path = file_path_no_extension + ext + '.info.json'; + + if (fs.existsSync(json_path)) fs.unlinkSync(json_path); + if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); +} + +async function removeIDFromArchive(archive_path, id) { + let data = await fs.readFile(archive_path, {encoding: 'utf-8'}); + if (!data) { + logger.error('Archive could not be found.'); + return; + } + + let dataArray = data.split('\n'); // convert file data in an array + const searchKeyword = id; // we are looking for a line, contains, key word id in the file + let lastIndex = -1; // let say, we have not found the keyword + + for (let index=0; index= 0; i--) { + num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i)); + } + return num_sum; +} + +function getMatchingCategoryFiles(category, files) { + return files && files.filter(file => file.category && file.category.uid === category.uid); +} + +function addUIDsToCategory(category, files) { + const files_that_match = getMatchingCategoryFiles(category, files); + category['uids'] = files_that_match.map(file => file.uid); + return files_that_match; +} async function recFindByExt(base,ext,files,result) { @@ -193,6 +343,16 @@ function removeFileExtension(filename) { return filename_parts.join('.'); } +/** + * setTimeout, but its a promise. + * @param {number} ms + */ + async function wait(ms) { + await new Promise(resolve => { + setTimeout(resolve, ms); + }); +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { @@ -215,13 +375,23 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p module.exports = { getJSONMp3: getJSONMp3, getJSONMp4: getJSONMp4, + getJSON: getJSON, getTrueFileName: getTrueFileName, getDownloadedThumbnail: getDownloadedThumbnail, + getDownloadedThumbnail2: getDownloadedThumbnail2, getExpectedFileSize: getExpectedFileSize, fixVideoMetadataPerms: fixVideoMetadataPerms, + fixVideoMetadataPerms2: fixVideoMetadataPerms2, deleteJSONFile: deleteJSONFile, + deleteJSONFile2: deleteJSONFile2, + removeIDFromArchive, removeIDFromArchive, getDownloadedFilesByType: getDownloadedFilesByType, + createContainerZipFile: createContainerZipFile, + durationStringToNumber: durationStringToNumber, + getMatchingCategoryFiles: getMatchingCategoryFiles, + addUIDsToCategory: addUIDsToCategory, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, + wait: wait, File: File } diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 0000000..a86aa60 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: youtubedl-material +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "4.2" diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt new file mode 100644 index 0000000..bf07841 --- /dev/null +++ b/chart/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "youtubedl-material.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "youtubedl-material.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "youtubedl-material.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "youtubedl-material.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl new file mode 100644 index 0000000..883b89e --- /dev/null +++ b/chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "youtubedl-material.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "youtubedl-material.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "youtubedl-material.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "youtubedl-material.labels" -}} +helm.sh/chart: {{ include "youtubedl-material.chart" . }} +{{ include "youtubedl-material.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "youtubedl-material.selectorLabels" -}} +app.kubernetes.io/name: {{ include "youtubedl-material.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "youtubedl-material.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "youtubedl-material.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/templates/appdata-pvc.yaml b/chart/templates/appdata-pvc.yaml new file mode 100644 index 0000000..e426650 --- /dev/null +++ b/chart/templates/appdata-pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.appdata.enabled (not .Values.persistence.appdata.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "youtubedl-material.fullname" . }}-appdata + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.appdata.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.appdata.size | quote }} + {{- if .Values.persistence.appdata.storageClass }} + {{- if (eq "-" .Values.persistence.appdata.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.appdata.storageClass }}" + {{- end }} + {{- end }} + {{- end -}} diff --git a/chart/templates/audio-pvc.yaml b/chart/templates/audio-pvc.yaml new file mode 100644 index 0000000..2de3d03 --- /dev/null +++ b/chart/templates/audio-pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.audio.enabled (not .Values.persistence.audio.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "youtubedl-material.fullname" . }}-audio + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.audio.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.audio.size | quote }} + {{- if .Values.persistence.audio.storageClass }} + {{- if (eq "-" .Values.persistence.audio.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.audio.storageClass }}" + {{- end }} + {{- end }} + {{- end -}} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml new file mode 100644 index 0000000..4d37b75 --- /dev/null +++ b/chart/templates/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "youtubedl-material.fullname" . }} + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "youtubedl-material.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "youtubedl-material.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "youtubedl-material.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 17442 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /app/appdata + name: appdata + {{- if .Values.persistence.appdata.subPath }} + subPath: {{ .Values.persistence.appdata.subPath }} + {{- end }} + - mountPath: /app/audio + name: audio + {{- if .Values.persistence.audio.subPath }} + subPath: {{ .Values.persistence.audio.subPath }} + {{- end }} + - mountPath: /app/video + name: video + {{- if .Values.persistence.video.subPath }} + subPath: {{ .Values.persistence.video.subPath }} + {{- end }} + - mountPath: /app/subscriptions + name: subscriptions + {{- if .Values.persistence.subscriptions.subPath }} + subPath: {{ .Values.persistence.subscriptions.subPath }} + {{- end }} + - mountPath: /app/users + name: users + {{- if .Values.persistence.users.subPath }} + subPath: {{ .Values.persistence.users.subPath }} + {{- end }} + volumes: + - name: appdata + {{- if .Values.persistence.appdata.enabled}} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.appdata.existingClaim }}{{ .Values.persistence.appdata.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-appdata{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + - name: audio + {{- if .Values.persistence.audio.enabled}} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.audio.existingClaim }}{{ .Values.persistence.audio.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-audio{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + - name: subscriptions + {{- if .Values.persistence.subscriptions.enabled}} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.subscriptions.existingClaim }}{{ .Values.persistence.subscriptions.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-subscriptions{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + - name: users + {{- if .Values.persistence.users.enabled}} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.users.existingClaim }}{{ .Values.persistence.users.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-users{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + - name: video + {{- if .Values.persistence.video.enabled}} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.video.existingClaim }}{{ .Values.persistence.video.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-video{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml new file mode 100644 index 0000000..79b9ece --- /dev/null +++ b/chart/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "youtubedl-material.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml new file mode 100644 index 0000000..01df5d2 --- /dev/null +++ b/chart/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "youtubedl-material.fullname" . }} + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "youtubedl-material.selectorLabels" . | nindent 4 }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..e04cc5e --- /dev/null +++ b/chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "youtubedl-material.serviceAccountName" . }} + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/subscriptions-pvc.yaml b/chart/templates/subscriptions-pvc.yaml new file mode 100644 index 0000000..ad5768c --- /dev/null +++ b/chart/templates/subscriptions-pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.subscriptions.enabled (not .Values.persistence.subscriptions.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "youtubedl-material.fullname" . }}-subscriptions + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.subscriptions.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.subscriptions.size | quote }} + {{- if .Values.persistence.subscriptions.storageClass }} + {{- if (eq "-" .Values.persistence.subscriptions.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.subscriptions.storageClass }}" + {{- end }} + {{- end }} + {{- end -}} diff --git a/chart/templates/tests/test-connection.yaml b/chart/templates/tests/test-connection.yaml new file mode 100644 index 0000000..3e4b1ba --- /dev/null +++ b/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "youtubedl-material.fullname" . }}-test-connection" + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "youtubedl-material.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/chart/templates/users-pvc.yaml b/chart/templates/users-pvc.yaml new file mode 100644 index 0000000..c12c116 --- /dev/null +++ b/chart/templates/users-pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.users.enabled (not .Values.persistence.users.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "youtubedl-material.fullname" . }}-users + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.users.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.users.size | quote }} + {{- if .Values.persistence.users.storageClass }} + {{- if (eq "-" .Values.persistence.users.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.users.storageClass }}" + {{- end }} + {{- end }} + {{- end -}} diff --git a/chart/templates/video-pvc.yaml b/chart/templates/video-pvc.yaml new file mode 100644 index 0000000..92718ee --- /dev/null +++ b/chart/templates/video-pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.video.enabled (not .Values.persistence.video.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "youtubedl-material.fullname" . }}-video + labels: + {{- include "youtubedl-material.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.video.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.video.size | quote }} + {{- if .Values.persistence.video.storageClass }} + {{- if (eq "-" .Values.persistence.video.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.video.storageClass }}" + {{- end }} + {{- end }} + {{- end -}} diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 0000000..b192e16 --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,153 @@ +# Default values for youtubedl-material. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: tzahi12345/youtubedl-material + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 17442 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +persistence: + appdata: + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + # subPath: some-subpath + accessMode: ReadWriteOnce + size: 1Gi + audio: + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + # subPath: some-subpath + accessMode: ReadWriteOnce + size: 50Gi + video: + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + # subPath: some-subpath + accessMode: ReadWriteOnce + size: 50Gi + subscriptions: + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + # subPath: some-subpath + accessMode: ReadWriteOnce + size: 50Gi + users: + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + # subPath: some-subpath + accessMode: ReadWriteOnce + size: 50Gi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/docker-compose.yml b/docker-compose.yml index 57c16d9..f2f0b93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ services: ytdl_material: environment: ALLOW_CONFIG_MUTATIONS: 'true' + ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27018' + ytdl_use_local_db: 'false' restart: always volumes: - ./appdata:/app/appdata @@ -12,4 +14,13 @@ services: - ./users:/app/users ports: - "8998:17442" - image: tzahi12345/youtubedl-material:latest \ No newline at end of file + image: tzahi12345/youtubedl-material:latest + ytdl-mongo-db: + image: mongo + ports: + - "27018:27017" + logging: + driver: "none" + container_name: mongo-db + volumes: + - ./db/:/data/db \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f0e3621..61b14b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "youtube-dl-material", - "version": "4.1.0", + "version": "4.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -180,9 +180,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "semver": { @@ -316,6 +316,12 @@ "ms": "2.1.2" } }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true + }, "resolve": { "version": "1.18.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.18.1.tgz", @@ -432,9 +438,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "semver": { @@ -705,9 +711,9 @@ }, "dependencies": { "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true } } @@ -784,9 +790,9 @@ }, "dependencies": { "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true } } @@ -1592,9 +1598,9 @@ }, "dependencies": { "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" } } }, @@ -1609,9 +1615,9 @@ }, "dependencies": { "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" } } }, @@ -1760,6 +1766,12 @@ "semver-intersect": "1.4.0" }, "dependencies": { + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true + }, "semver": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", @@ -2616,15 +2628,6 @@ "tweetnacl": "^0.14.3" } }, - "better-assert": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz", - "integrity": "sha1-QIZrnhueC1W0gYlDEeaPr/rrxSI=", - "dev": true, - "requires": { - "callsite": "1.0.0" - } - }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -3059,12 +3062,6 @@ "caller-callsite": "^2.0.0" } }, - "callsite": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", - "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA=", - "dev": true - }, "callsites": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", @@ -4513,24 +4510,24 @@ "dev": true }, "elliptic": { - "version": "6.5.3", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", - "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", "dev": true, "requires": { - "bn.js": "^4.4.0", - "brorand": "^1.0.1", + "bn.js": "^4.11.9", + "brorand": "^1.1.0", "hash.js": "^1.0.0", - "hmac-drbg": "^1.0.0", - "inherits": "^2.0.1", - "minimalistic-assert": "^1.0.0", - "minimalistic-crypto-utils": "^1.0.0" + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true } } @@ -4582,37 +4579,37 @@ } }, "engine.io": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.4.2.tgz", - "integrity": "sha512-b4Q85dFkGw+TqgytGPrGgACRUhsdKc9S9ErRAXpPGy/CXKs4tYoHDkvIRdsseAF7NjfVwjRFIn6KTnbw7LwJZg==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.5.0.tgz", + "integrity": "sha512-21HlvPUKaitDGE4GXNtQ7PLP0Sz4aWLddMPw2VTyFz1FVZqu/kZsJUO8WNpKuE/OCL7nkfRaOui2ZCJloGznGA==", "dev": true, "requires": { "accepts": "~1.3.4", "base64id": "2.0.0", - "cookie": "0.3.1", + "cookie": "~0.4.1", "debug": "~4.1.0", "engine.io-parser": "~2.2.0", - "ws": "^7.1.2" + "ws": "~7.4.2" }, "dependencies": { "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==", "dev": true }, "ws": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.1.tgz", - "integrity": "sha512-pTsP8UAfhy3sk1lSk/O/s4tjD0CRwvMnzvwr4OKGX7ZvqZtUyx4KIJB5JWbkykPoc55tixMGgTNoh3k4FkNGFQ==", + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", + "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", "dev": true } } }, "engine.io-client": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.4.4.tgz", - "integrity": "sha512-iU4CRr38Fecj8HoZEnFtm2EiKGbYZcPn3cHxqNGl/tmdWRf60KhK+9vE0JeSjgnlS/0oynEfLgKbT9ALpim0sQ==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.5.2.tgz", + "integrity": "sha512-QEqIp+gJ/kMHeUun7f5Vv3bteRHppHH/FMBQX/esFj/fuYfjyUKWGMo3VCvIP/V8bE9KcjHmRZrhIz2Z9oNsDA==", "dev": true, "requires": { "component-emitter": "~1.3.0", @@ -4623,8 +4620,8 @@ "indexof": "0.0.1", "parseqs": "0.0.6", "parseuri": "0.0.6", - "ws": "~6.1.0", - "xmlhttprequest-ssl": "~1.5.4", + "ws": "~7.4.2", + "xmlhttprequest-ssl": "~1.6.2", "yeast": "0.1.2" }, "dependencies": { @@ -4643,26 +4640,11 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, - "parseqs": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz", - "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w==", - "dev": true - }, - "parseuri": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz", - "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow==", - "dev": true - }, "ws": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", - "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", - "dev": true, - "requires": { - "async-limiter": "~1.0.0" - } + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", + "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==", + "dev": true } } }, @@ -5930,9 +5912,9 @@ } }, "hosted-git-info": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.7.tgz", - "integrity": "sha512-fWqc0IcuXs+BmE9orLDyVykAG9GJtGLGuZAAqgcckPgv5xad4AcXGIv8galtQvlwutxSlaMcdw7BUtq2EIvqCQ==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.8.tgz", + "integrity": "sha512-aXpmwoOhRBrw6X3j0h5RloK4x1OzsxMPyxqIHyNfSe2pypkVTZFpEiRoSipPEPlMrh0HW/XsjkJ5WgnCirpNUw==", "dev": true, "requires": { "lru-cache": "^6.0.0" @@ -6338,9 +6320,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "dev": true }, "inquirer": { @@ -6405,9 +6387,9 @@ "dev": true }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "supports-color": { @@ -7498,9 +7480,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.memoize": { "version": "4.1.2", @@ -7719,9 +7701,9 @@ } }, "ssri": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", - "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", + "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", "dev": true, "requires": { "figgy-pudding": "^3.5.1" @@ -7775,6 +7757,11 @@ } } }, + "material-icons": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/material-icons/-/material-icons-0.5.4.tgz", + "integrity": "sha512-5ycazkNmIOtV78Ff3WgvxQESoJuujdRm0cNbf18fmyJN20jHyqp9rpwi4EfQyGimag0ZLElxtVg3H9enIKdOOw==" + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -8284,9 +8271,9 @@ }, "dependencies": { "hosted-git-info": { - "version": "2.8.8", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", - "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true } } @@ -8435,9 +8422,9 @@ }, "dependencies": { "hosted-git-info": { - "version": "2.8.8", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", - "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, "lru-cache": { @@ -8511,12 +8498,6 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true }, - "object-component": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz", - "integrity": "sha1-8MaapQ78lbhmwYb0AKM3acsvEpE=", - "dev": true - }, "object-copy": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", @@ -8937,9 +8918,9 @@ } }, "hosted-git-info": { - "version": "2.8.8", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", - "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, "lru-cache": { @@ -9003,9 +8984,9 @@ } }, "ssri": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", - "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", + "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", "dev": true, "requires": { "figgy-pudding": "^3.5.1" @@ -9139,22 +9120,16 @@ } }, "parseqs": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", - "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=", - "dev": true, - "requires": { - "better-assert": "~1.0.0" - } + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz", + "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w==", + "dev": true }, "parseuri": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz", - "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=", - "dev": true, - "requires": { - "better-assert": "~1.0.0" - } + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz", + "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow==", + "dev": true }, "parseurl": { "version": "1.3.3", @@ -11672,16 +11647,16 @@ } }, "socket.io": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.3.0.tgz", - "integrity": "sha512-2A892lrj0GcgR/9Qk81EaY2gYhCBxurV0PfmmESO6p27QPrUK1J3zdns+5QPqvUYK2q657nSj0guoIil9+7eFg==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.4.1.tgz", + "integrity": "sha512-Si18v0mMXGAqLqCVpTxBa8MGqriHGQh8ccEOhmsmNS3thNCGBwO8WGrwMibANsWtQQ5NStdZwHqZR3naJVFc3w==", "dev": true, "requires": { "debug": "~4.1.0", - "engine.io": "~3.4.0", + "engine.io": "~3.5.0", "has-binary2": "~1.0.2", "socket.io-adapter": "~1.1.0", - "socket.io-client": "2.3.0", + "socket.io-client": "2.4.0", "socket.io-parser": "~3.4.0" } }, @@ -11692,38 +11667,32 @@ "dev": true }, "socket.io-client": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.3.0.tgz", - "integrity": "sha512-cEQQf24gET3rfhxZ2jJ5xzAOo/xhZwK+mOqtGRg5IowZsMgwvHwnf/mCRapAAkadhM26y+iydgwsXGObBB5ZdA==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.4.0.tgz", + "integrity": "sha512-M6xhnKQHuuZd4Ba9vltCLT9oa+YvTsP8j9NcEiLElfIg8KeYPyhWOes6x4t+LTAC8enQbE/995AdTem2uNyKKQ==", "dev": true, "requires": { "backo2": "1.0.2", - "base64-arraybuffer": "0.1.5", "component-bind": "1.0.0", - "component-emitter": "1.2.1", - "debug": "~4.1.0", - "engine.io-client": "~3.4.0", + "component-emitter": "~1.3.0", + "debug": "~3.1.0", + "engine.io-client": "~3.5.0", "has-binary2": "~1.0.2", - "has-cors": "1.1.0", "indexof": "0.0.1", - "object-component": "0.0.3", - "parseqs": "0.0.5", - "parseuri": "0.0.5", + "parseqs": "0.0.6", + "parseuri": "0.0.6", "socket.io-parser": "~3.3.0", "to-array": "0.1.4" }, "dependencies": { - "base64-arraybuffer": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz", - "integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg=", - "dev": true - }, - "component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=", - "dev": true + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, + "requires": { + "ms": "2.0.0" + } }, "isarray": { "version": "2.0.1", @@ -11738,31 +11707,14 @@ "dev": true }, "socket.io-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.3.1.tgz", - "integrity": "sha512-1QLvVAe8dTz+mKmZ07Swxt+LAo4Y1ff50rlyoEx00TQmDFVQYPfcqGvIDJLGaBdhdNCecXtyKpD+EgKGcmmbuQ==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.3.2.tgz", + "integrity": "sha512-FJvDBuOALxdCI9qwRrO/Rfp9yfndRtc1jSgVgV8FDraihmSP/MLGD5PEuJrNfjALvcQ+vMDM/33AWOYP/JSjDg==", "dev": true, "requires": { "component-emitter": "~1.3.0", "debug": "~3.1.0", "isarray": "2.0.1" - }, - "dependencies": { - "component-emitter": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", - "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", - "dev": true - }, - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } } } } @@ -12077,9 +12029,9 @@ } }, "ssri": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.0.tgz", - "integrity": "sha512-aq/pz989nxVYwn16Tsbj1TqFpD5LLrQxHf5zaHuieFV+R0Bbr4y8qUsOA45hXT/N4/9UNXTarBjnjVmjSOVaAA==", + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", "dev": true, "requires": { "minipass": "^3.1.1" @@ -13069,9 +13021,9 @@ } }, "url-parse": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", - "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.1.tgz", + "integrity": "sha512-HOfCOUJt7iSYzEx/UqgtwKRMC6EU91NFhsCHMv9oM03VJcVo2Qrp8T8kI9D7amFf1cu+/3CEhgb3rF9zL7k85Q==", "dev": true, "requires": { "querystringify": "^2.1.1", @@ -13791,8 +13743,7 @@ }, "ssri": { "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", - "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "resolved": "", "dev": true, "requires": { "figgy-pudding": "^3.5.1" @@ -14533,9 +14484,9 @@ "dev": true }, "xmlhttprequest-ssl": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz", - "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=", + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.6.2.tgz", + "integrity": "sha512-tYOaldF/0BLfKuoA39QMwD4j2m8lq4DIncqj1yuNELX4vz9+z/ieG/vwmctjJce+boFHXstqhWnHSxc4W8f4qg==", "dev": true }, "xtend": { diff --git a/package.json b/package.json index ab15811..e55369d 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "file-saver": "^2.0.2", "filesize": "^6.1.0", "fingerprintjs2": "^2.1.0", + "material-icons": "^0.5.4", "nan": "^2.14.1", "ng-lazyload-image": "^7.0.1", "ngx-avatar": "^4.0.0", diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 0ae868f..ec638bd 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -86,6 +86,7 @@ import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component'; import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.component'; import { H401Interceptor } from './http.interceptor'; +import { ConcurrentStreamComponent } from './components/concurrent-stream/concurrent-stream.component'; registerLocaleData(es, 'es'); @@ -134,7 +135,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible CustomPlaylistsComponent, EditCategoryDialogComponent, TwitchChatComponent, - SeeMoreComponent + SeeMoreComponent, + ConcurrentStreamComponent ], imports: [ CommonModule, diff --git a/src/app/components/concurrent-stream/concurrent-stream.component.html b/src/app/components/concurrent-stream/concurrent-stream.component.html new file mode 100644 index 0000000..414c4ac --- /dev/null +++ b/src/app/components/concurrent-stream/concurrent-stream.component.html @@ -0,0 +1,6 @@ +
+ + + + +
\ No newline at end of file diff --git a/src/app/components/concurrent-stream/concurrent-stream.component.scss b/src/app/components/concurrent-stream/concurrent-stream.component.scss new file mode 100644 index 0000000..d3b74be --- /dev/null +++ b/src/app/components/concurrent-stream/concurrent-stream.component.scss @@ -0,0 +1,7 @@ +.buttons-container { + display: flex; + align-items: center; + justify-content: center; + margin-top: 15px; + margin-bottom: 15px; +} \ No newline at end of file diff --git a/src/app/components/concurrent-stream/concurrent-stream.component.spec.ts b/src/app/components/concurrent-stream/concurrent-stream.component.spec.ts new file mode 100644 index 0000000..a881ec8 --- /dev/null +++ b/src/app/components/concurrent-stream/concurrent-stream.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ConcurrentStreamComponent } from './concurrent-stream.component'; + +describe('ConcurrentStreamComponent', () => { + let component: ConcurrentStreamComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ ConcurrentStreamComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(ConcurrentStreamComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/concurrent-stream/concurrent-stream.component.ts b/src/app/components/concurrent-stream/concurrent-stream.component.ts new file mode 100644 index 0000000..6c2cc67 --- /dev/null +++ b/src/app/components/concurrent-stream/concurrent-stream.component.ts @@ -0,0 +1,140 @@ +import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-concurrent-stream', + templateUrl: './concurrent-stream.component.html', + styleUrls: ['./concurrent-stream.component.scss'] +}) +export class ConcurrentStreamComponent implements OnInit { + + @Input() server_mode = false; + @Input() playback_timestamp; + @Input() playing; + @Input() uid; + + @Output() setPlaybackTimestamp = new EventEmitter(); + @Output() togglePlayback = new EventEmitter(); + @Output() setPlaybackRate = new EventEmitter(); + + started = false; + server_started = false; + watch_together_clicked = false; + + server_already_exists = null; + + check_timeout: any; + update_timeout: any; + + PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_PLAYBACK_MODIFICATION = 0.5; + PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_SKIP = 2; + + PLAYBACK_MODIFIER = 0.1; + + playback_rate_modified = false; + + constructor(private postsService: PostsService) { } + + // flow: click start watching -> check for available stream to enable join button and if user, display "start stream" + // users who join a stream will send continuous requests for info on playback + + ngOnInit(): void { + + } + + startServer() { + this.started = true; + this.server_started = true; + this.update_timeout = setInterval(() => { + this.updateStream(); + }, 1000); + } + + updateStream() { + this.postsService.updateConcurrentStream(this.uid, this.playback_timestamp, Date.now()/1000, this.playing).subscribe(res => { + }); + } + + startClient() { + this.started = true; + } + + checkStream() { + if (this.server_started) { return; } + const current_playback_timestamp = this.playback_timestamp; + const current_unix_timestamp = Date.now()/1000; + this.postsService.checkConcurrentStream(this.uid).subscribe(res => { + const stream = res['stream']; + + if (!stream) { + this.server_already_exists = false; + return; + } + + this.server_already_exists = true; + + // check whether client has joined the stream + if (!this.started) { return; } + + if (!stream['playing'] && this.playing) { + // tell client to pause and set the timestamp to sync + this.togglePlayback.emit(false); + this.setPlaybackTimestamp.emit(stream['playback_timestamp']); + } else if (stream['playing']) { + // sync unpause state + if (!this.playing) { this.togglePlayback.emit(true); } + + // sync time + const zeroed_local_unix_timestamp = current_unix_timestamp - current_playback_timestamp; + const zeroed_server_unix_timestamp = stream['unix_timestamp'] - stream['playback_timestamp']; + + const seconds_behind_locally = zeroed_local_unix_timestamp - zeroed_server_unix_timestamp; + + if (Math.abs(seconds_behind_locally) > this.PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_SKIP) { + // skip to playback timestamp because the difference is too high + this.setPlaybackTimestamp.emit(this.playback_timestamp + seconds_behind_locally + 0.3); + this.playback_rate_modified = false; + } else if (!this.playback_rate_modified && Math.abs(seconds_behind_locally) > this.PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_PLAYBACK_MODIFICATION) { + // increase playback speed to avoid skipping + let seconds_to_wait = (Math.abs(seconds_behind_locally)/this.PLAYBACK_MODIFIER); + seconds_to_wait += 0.3/this.PLAYBACK_MODIFIER; + + this.playback_rate_modified = true; + + if (seconds_behind_locally > 0) { + // increase speed + this.setPlaybackRate.emit(1 + this.PLAYBACK_MODIFIER); + setTimeout(() => { + this.setPlaybackRate.emit(1); + this.playback_rate_modified = false; + }, seconds_to_wait * 1000); + } else { + // decrease speed + this.setPlaybackRate.emit(1 - this.PLAYBACK_MODIFIER); + setTimeout(() => { + this.setPlaybackRate.emit(1); + this.playback_rate_modified = false; + }, seconds_to_wait * 1000); + } + } + } + }); + } + + startWatching() { + this.watch_together_clicked = true; + this.check_timeout = setInterval(() => { + this.checkStream(); + }, 1000); + } + + stop() { + if (this.check_timeout) { clearInterval(this.check_timeout); } + if (this.update_timeout) { clearInterval(this.update_timeout); } + this.started = false; + this.server_started = false; + this.watch_together_clicked = false; + } + + +} diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index 73e3036..6071d3d 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -53,16 +53,15 @@ export class CustomPlaylistsComponent implements OnInit { goToPlaylist(info_obj) { const playlist = info_obj.file; const playlistID = playlist.id; - const type = playlist.type; if (playlist) { if (this.postsService.config['Extra']['download_only_mode']) { - this.downloading_content[type][playlistID] = true; - this.downloadPlaylist(playlist.fileNames, type, playlist.name, playlistID); + this.downloadPlaylist(playlist.id, playlist.name); } else { localStorage.setItem('player_navigator', this.router.url); - const fileNames = playlist.fileNames; - this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID, auto: playlist.auto}]); + const routeParams = {playlist_id: playlistID}; + if (playlist.auto) { routeParams['auto'] = playlist.auto; } + this.router.navigate(['/player', routeParams]); } } else { // playlist not found @@ -70,11 +69,12 @@ export class CustomPlaylistsComponent implements OnInit { } } - downloadPlaylist(fileNames, type, zipName = null, playlistID = null) { - this.postsService.downloadFileFromServer(fileNames, type, zipName).subscribe(res => { - if (playlistID) { this.downloading_content[type][playlistID] = false }; - const blob: Blob = res; - saveAs(blob, zipName + '.zip'); + downloadPlaylist(playlist_id, playlist_name) { + this.downloading_content[playlist_id] = true; + this.postsService.downloadPlaylistFromServer(playlist_id).subscribe(res => { + this.downloading_content[playlist_id] = false; + const blob: any = res; + saveAs(blob, playlist_name + '.zip'); }); } @@ -97,7 +97,7 @@ export class CustomPlaylistsComponent implements OnInit { const index = args.index; const dialogRef = this.dialog.open(ModifyPlaylistComponent, { data: { - playlist: playlist, + playlist_id: playlist.id, width: '65vw' } }); diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html index 8880b30..c6870b5 100644 --- a/src/app/components/downloads/downloads.component.html +++ b/src/app/components/downloads/downloads.component.html @@ -1,21 +1,21 @@
-
- +
+ -

Session ID: {{session_downloads.key}} -  (current) +

Session ID: {{session_downloads['session_id']}} +  (current)

-
- - +
+ +
- +
diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index d7e7d2a..1539448 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -35,7 +35,7 @@ import { Router } from '@angular/router'; export class DownloadsComponent implements OnInit, OnDestroy { downloads_check_interval = 1000; - downloads = {}; + downloads = []; interval_id = null; keys = Object.keys; @@ -137,6 +137,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { this.downloads[session_id] = session_downloads_by_id; } else { for (let j = 0; j < session_download_ids.length; j++) { + if (session_download_ids[j] === 'session_id' || session_download_ids[j] === '_id') continue; const download_id = session_download_ids[j]; const download = new_downloads_by_session[session_id][download_id] if (!this.downloads[session_id][download_id]) { @@ -156,11 +157,10 @@ export class DownloadsComponent implements OnInit, OnDestroy { downloadsValid() { let valid = false; - const keys = this.keys(this.downloads); - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; - const value = this.downloads[key]; - if (this.keys(value).length > 0) { + for (let i = 0; i < this.downloads.length; i++) { + const session_downloads = this.downloads[i]; + if (!session_downloads) continue; + if (this.keys(session_downloads).length > 2) { valid = true; break; } diff --git a/src/app/components/manage-role/manage-role.component.html b/src/app/components/manage-role/manage-role.component.html index b3e8de9..047dc20 100644 --- a/src/app/components/manage-role/manage-role.component.html +++ b/src/app/components/manage-role/manage-role.component.html @@ -5,7 +5,7 @@

{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}

- + Yes No diff --git a/src/app/components/manage-role/manage-role.component.ts b/src/app/components/manage-role/manage-role.component.ts index 4e05e29..dbe2511 100644 --- a/src/app/components/manage-role/manage-role.component.ts +++ b/src/app/components/manage-role/manage-role.component.ts @@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit { } changeRolePermissions(change, permission) { - this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => { + this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => { if (res['success']) { } else { diff --git a/src/app/components/modify-users/modify-users.component.html b/src/app/components/modify-users/modify-users.component.html index 31c2b80..cd054af 100644 --- a/src/app/components/modify-users/modify-users.component.html +++ b/src/app/components/modify-users/modify-users.component.html @@ -94,7 +94,7 @@
- +
diff --git a/src/app/components/modify-users/modify-users.component.ts b/src/app/components/modify-users/modify-users.component.ts index 9e54fda..e1b7a98 100644 --- a/src/app/components/modify-users/modify-users.component.ts +++ b/src/app/components/modify-users/modify-users.component.ts @@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit { getRoles() { this.postsService.getRoles().subscribe(res => { - this.roles = []; - const roles = res['roles']; - const role_names = Object.keys(roles); - for (let i = 0; i < role_names.length; i++) { - const role_name = role_names[i]; - this.roles.push({ - name: role_name, - permissions: roles[role_name]['permissions'] - }); - } + this.roles = res['roles']; }); } diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index 6aec7f2..144f3bd 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -166,15 +166,14 @@ export class RecentVideosComponent implements OnInit { const sub = this.postsService.getSubscriptionByID(file.sub_id); if (sub.streamingOnly) { // streaming only mode subscriptions - !new_tab ? this.router.navigate(['/player', {name: file.id, - url: file.requested_formats ? file.requested_formats[0].url : file.url}]) - : window.open(`/#/player;name=${file.id};url=${file.requested_formats ? file.requested_formats[0].url : file.url}`); + // !new_tab ? this.router.navigate(['/player', {name: file.id, + // url: file.requested_formats ? file.requested_formats[0].url : file.url}]) + // : window.open(`/#/player;name=${file.id};url=${file.requested_formats ? file.requested_formats[0].url : file.url}`); } else { // normal subscriptions - !new_tab ? this.router.navigate(['/player', {fileNames: file.id, - type: file.isAudio ? 'audio' : 'video', subscriptionName: sub.name, - subPlaylist: sub.isPlaylist}]) - : window.open(`/#/player;fileNames=${file.id};type=${file.isAudio ? 'audio' : 'video'};subscriptionName=${sub.name};subPlaylist=${sub.isPlaylist}`); + !new_tab ? this.router.navigate(['/player', {uid: file.uid, + type: file.isAudio ? 'audio' : 'video', sub_id: sub.id}]) + : window.open(`/#/player;uid=${file.uid};type=${file.isAudio ? 'audio' : 'video'};sub_id=${sub.id}`); } } else { // normal files @@ -201,8 +200,7 @@ export class RecentVideosComponent implements OnInit { const type = file.isAudio ? 'audio' : 'video'; const ext = type === 'audio' ? '.mp3' : '.mp4' const sub = this.postsService.getSubscriptionByID(file.sub_id); - this.postsService.downloadFileFromServer(file.id, type, null, null, sub.name, sub.isPlaylist, - this.postsService.user ? this.postsService.user.uid : null, null).subscribe(res => { + this.postsService.downloadFileFromServer(file.uid).subscribe(res => { const blob: Blob = res; saveAs(blob, file.id + ext); }, err => { @@ -215,14 +213,14 @@ export class RecentVideosComponent implements OnInit { const ext = type === 'audio' ? '.mp3' : '.mp4' const name = file.id; this.downloading_content[type][name] = true; - this.postsService.downloadFileFromServer(name, type).subscribe(res => { + this.postsService.downloadFileFromServer(file.uid).subscribe(res => { this.downloading_content[type][name] = false; const blob: Blob = res; saveAs(blob, decodeURIComponent(name) + ext); if (!this.postsService.config.Extra.file_manager_enabled) { // tell server to delete the file once downloaded - this.postsService.deleteFile(name, type).subscribe(delRes => { + this.postsService.deleteFile(file.uid).subscribe(delRes => { // reload mp4s this.getAllFiles(); }); @@ -245,7 +243,7 @@ export class RecentVideosComponent implements OnInit { } deleteNormalFile(file, blacklistMode = false) { - this.postsService.deleteFile(file.uid, file.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { + this.postsService.deleteFile(file.uid, blacklistMode).subscribe(result => { if (result) { this.postsService.openSnackBar('Delete success!', 'OK.'); this.removeFileCard(file); diff --git a/src/app/create-playlist/create-playlist.component.html b/src/app/create-playlist/create-playlist.component.html index 8027983..ad21ec0 100644 --- a/src/app/create-playlist/create-playlist.component.html +++ b/src/app/create-playlist/create-playlist.component.html @@ -19,9 +19,9 @@ Audio files Videos - {{file.id}} - {{file.id}} - {{file.id}} + {{file.id}} + {{file.id}} + {{file.id}} diff --git a/src/app/create-playlist/create-playlist.component.ts b/src/app/create-playlist/create-playlist.component.ts index b9cf976..c22d32d 100644 --- a/src/app/create-playlist/create-playlist.component.ts +++ b/src/app/create-playlist/create-playlist.component.ts @@ -51,9 +51,8 @@ export class CreatePlaylistComponent implements OnInit { createPlaylist() { const thumbnailURL = this.getThumbnailURL(); - const duration = this.calculateDuration(); this.create_in_progress = true; - this.postsService.createPlaylist(this.name, this.filesSelect.value, this.type, thumbnailURL, duration).subscribe(res => { + this.postsService.createPlaylist(this.name, this.filesSelect.value, this.type, thumbnailURL).subscribe(res => { this.create_in_progress = false; if (res['success']) { this.dialogRef.close(true); @@ -78,36 +77,4 @@ export class CreatePlaylistComponent implements OnInit { } return null; } - - getDuration(file_id) { - let properFilesToSelectFrom = this.filesToSelectFrom; - if (!this.filesToSelectFrom) { - properFilesToSelectFrom = this.type === 'audio' ? this.audiosToSelectFrom : this.videosToSelectFrom; - } - for (let i = 0; i < properFilesToSelectFrom.length; i++) { - const file = properFilesToSelectFrom[i]; - if (file.id === file_id) { - return file.duration; - } - } - return null; - } - - calculateDuration() { - let sum = 0; - for (let i = 0; i < this.filesSelect.value.length; i++) { - const duration_val = this.getDuration(this.filesSelect.value[i]); - sum += typeof duration_val === 'string' ? this.durationStringToNumber(duration_val) : duration_val; - } - return sum; - } - - durationStringToNumber(dur_str) { - let num_sum = 0; - const dur_str_parts = dur_str.split(':'); - for (let i = dur_str_parts.length-1; i >= 0; i--) { - num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i)); - } - return num_sum; - } } diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.html b/src/app/dialogs/modify-playlist/modify-playlist.component.html index 69f4cad..a8471bb 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.html +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.html @@ -1,38 +1,40 @@

Modify playlist

- -
- - - -
- -
-
- Normal order  - Reverse order  - +
+ +
+ + +
-
- -
-
+
+
+ Normal order  + Reverse order  + +
- - - -
{{playlist_item}}
 - - - - - +
+ +
+
+ + + + +
{{playlist_item.title}}
 + + + + + +
- + \ No newline at end of file diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.ts b/src/app/dialogs/modify-playlist/modify-playlist.component.ts index 414fc92..b482bb8 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts @@ -10,8 +10,12 @@ import { PostsService } from 'app/posts.services'; }) export class ModifyPlaylistComponent implements OnInit { + playlist_id = null; + original_playlist = null; playlist = null; + playlist_file_objs = null; + available_files = []; all_files = []; playlist_updated = false; @@ -23,9 +27,8 @@ export class ModifyPlaylistComponent implements OnInit { ngOnInit(): void { if (this.data) { - this.playlist = JSON.parse(JSON.stringify(this.data.playlist)); - this.original_playlist = JSON.parse(JSON.stringify(this.data.playlist)); - this.getFiles(); + this.playlist_id = this.data.playlist_id; + this.getPlaylist(); } this.reverse_order = localStorage.getItem('default_playlist_order_reversed') === 'true'; @@ -44,11 +47,12 @@ export class ModifyPlaylistComponent implements OnInit { } processFiles(new_files = null) { - if (new_files) { this.all_files = new_files.map(file => file.id); } - this.available_files = this.all_files.filter(e => !this.playlist.fileNames.includes(e)) + if (new_files) { this.all_files = new_files; } + this.available_files = this.all_files.filter(e => !this.playlist_file_objs.includes(e)) } updatePlaylist() { + this.playlist['uids'] = this.playlist_file_objs.map(playlist_file_obj => playlist_file_obj['uid']) this.postsService.updatePlaylist(this.playlist).subscribe(res => { this.playlist_updated = true; this.postsService.openSnackBar('Playlist updated successfully.'); @@ -57,28 +61,30 @@ export class ModifyPlaylistComponent implements OnInit { } playlistChanged() { - return JSON.stringify(this.playlist) === JSON.stringify(this.original_playlist); + return JSON.stringify(this.playlist) !== JSON.stringify(this.original_playlist); } getPlaylist() { - this.postsService.getPlaylist(this.playlist.id, this.playlist.type, null).subscribe(res => { + this.postsService.getPlaylist(this.playlist_id, null, true).subscribe(res => { if (res['playlist']) { this.playlist = res['playlist']; + this.playlist_file_objs = res['file_objs']; this.original_playlist = JSON.parse(JSON.stringify(this.playlist)); + this.getFiles(); } }); } addContent(file) { - this.playlist.fileNames.push(file); + this.playlist_file_objs.push(file); this.processFiles(); } removeContent(index) { if (this.reverse_order) { - index = this.playlist.fileNames.length - 1 - index; + index = this.playlist_file_objs.length - 1 - index; } - this.playlist.fileNames.splice(index, 1); + this.playlist_file_objs.splice(index, 1); this.processFiles(); } @@ -89,10 +95,10 @@ export class ModifyPlaylistComponent implements OnInit { drop(event: CdkDragDrop) { if (this.reverse_order) { - event.previousIndex = this.playlist.fileNames.length - 1 - event.previousIndex; - event.currentIndex = this.playlist.fileNames.length - 1 - event.currentIndex; + event.previousIndex = this.playlist_file_objs.length - 1 - event.previousIndex; + event.currentIndex = this.playlist_file_objs.length - 1 - event.currentIndex; } - moveItemInArray(this.playlist.fileNames, event.previousIndex, event.currentIndex); + moveItemInArray(this.playlist_file_objs, event.previousIndex, event.currentIndex); } } diff --git a/src/app/dialogs/share-media-dialog/share-media-dialog.component.html b/src/app/dialogs/share-media-dialog/share-media-dialog.component.html index fcd8f3c..7175b52 100644 --- a/src/app/dialogs/share-media-dialog/share-media-dialog.component.html +++ b/src/app/dialogs/share-media-dialog/share-media-dialog.component.html @@ -1,7 +1,6 @@

Share playlist - Share video - Share audio + Share file

diff --git a/src/app/dialogs/share-media-dialog/share-media-dialog.component.ts b/src/app/dialogs/share-media-dialog/share-media-dialog.component.ts index 137e8fc..332a461 100644 --- a/src/app/dialogs/share-media-dialog/share-media-dialog.component.ts +++ b/src/app/dialogs/share-media-dialog/share-media-dialog.component.ts @@ -11,7 +11,6 @@ import { PostsService } from 'app/posts.services'; }) export class ShareMediaDialogComponent implements OnInit { - type = null; uid = null; uuid = null; share_url = null; @@ -26,14 +25,13 @@ export class ShareMediaDialogComponent implements OnInit { ngOnInit(): void { if (this.data) { - this.type = this.data.type; this.uid = this.data.uid; this.uuid = this.data.uuid; this.sharing_enabled = this.data.sharing_enabled; this.is_playlist = this.data.is_playlist; this.current_timestamp = (this.data.current_timestamp / 1000).toFixed(2); - const arg = (this.is_playlist ? ';id=' : ';uid='); + const arg = (this.is_playlist ? ';playlist_id=' : ';uid='); this.default_share_url = window.location.href.split(';')[0] + arg + this.uid; if (this.uuid) { this.default_share_url += ';uuid=' + this.uuid; @@ -65,7 +63,7 @@ export class ShareMediaDialogComponent implements OnInit { sharingChanged(event) { if (event.checked) { - this.postsService.enableSharing(this.uid, this.type, this.is_playlist).subscribe(res => { + this.postsService.enableSharing(this.uid, this.is_playlist).subscribe(res => { if (res['success']) { this.openSnackBar('Sharing enabled.'); this.sharing_enabled = true; @@ -76,7 +74,7 @@ export class ShareMediaDialogComponent implements OnInit { this.openSnackBar('Failed to enable sharing - server error.'); }); } else { - this.postsService.disableSharing(this.uid, this.type, this.is_playlist).subscribe(res => { + this.postsService.disableSharing(this.uid, this.is_playlist).subscribe(res => { if (res['success']) { this.openSnackBar('Sharing disabled.'); this.sharing_enabled = false; diff --git a/src/app/file-card/file-card.component.ts b/src/app/file-card/file-card.component.ts index 68a8453..5596eec 100644 --- a/src/app/file-card/file-card.component.ts +++ b/src/app/file-card/file-card.component.ts @@ -56,7 +56,7 @@ export class FileCardComponent implements OnInit { deleteFile(blacklistMode = false) { if (!this.playlist) { - this.postsService.deleteFile(this.uid, this.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { + this.postsService.deleteFile(this.uid, blacklistMode).subscribe(result => { if (result) { this.openSnackBar('Delete success!', 'OK.'); this.removeFile.emit(this.name); @@ -84,7 +84,7 @@ export class FileCardComponent implements OnInit { editPlaylistDialog() { const dialogRef = this.dialog.open(ModifyPlaylistComponent, { data: { - playlist: this.playlist, + playlist_id: this.playlist.id, width: '65vw' } }); diff --git a/src/app/http.interceptor.ts b/src/app/http.interceptor.ts index edde22b..b941a34 100644 --- a/src/app/http.interceptor.ts +++ b/src/app/http.interceptor.ts @@ -14,7 +14,7 @@ export class H401Interceptor implements HttpInterceptor { return next.handle(request).pipe(catchError(err => { if (err.status === 401) { localStorage.setItem('jwt_token', null); - if (this.router.url !== '/login') { + if (this.router.url !== '/login' && !this.router.url.includes('player')) { this.router.navigate(['/login']).then(() => { this.openSnackBar('Login expired, please login again.'); }); diff --git a/src/app/main/main.component.css b/src/app/main/main.component.css index 4ee7412..8411370 100644 --- a/src/app/main/main.component.css +++ b/src/app/main/main.component.css @@ -124,6 +124,10 @@ mat-form-field.mat-form-field { width: 100%; } +.advanced-input-time { + margin-left: 10px; +} + .edit-button { margin-left: 10px; top: -5px; diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index 9cb905e..2d4ddb0 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -20,11 +20,16 @@ - - - {{option.label}} + + Max - + + + + {{option.key}} + + + -
+
Use authentication @@ -139,11 +144,26 @@
-
+
+
+ + + Crop file + + + + + +
+
+ + + +
diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 30694bf..107c311 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -54,6 +54,9 @@ export class MainComponent implements OnInit { youtubeAuthEnabled = false; youtubeUsername = null; youtubePassword = null; + cropFile = false; + cropFileStart = null; + cropFileEnd = null; urlError = false; path = ''; url = ''; @@ -339,12 +342,8 @@ export class MainComponent implements OnInit { } } - public goToFile(name, isAudio, uid) { - if (isAudio) { - this.downloadHelperMp3(name, uid, false, false, null, true); - } else { - this.downloadHelperMp4(name, uid, false, false, null, true); - } + public goToFile(container, isAudio, uid) { + this.downloadHelper(container, isAudio ? 'audio' : 'video', false, false, null, true); } public goToPlaylist(playlistID, type) { @@ -352,7 +351,7 @@ export class MainComponent implements OnInit { if (playlist) { if (this.downloadOnlyMode) { this.downloading_content[type][playlistID] = true; - this.downloadPlaylist(playlist.fileNames, type, playlist.name, playlistID); + this.downloadPlaylist(playlist); } else { localStorage.setItem('player_navigator', this.router.url); const fileNames = playlist.fileNames; @@ -376,56 +375,26 @@ export class MainComponent implements OnInit { // download helpers - downloadHelperMp3(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { + downloadHelper(container, type, is_playlist = false, force_view = false, new_download = null, navigate_mode = false) { this.downloadingfile = false; if (this.multiDownloadMode && !this.downloadOnlyMode && !navigate_mode) { // do nothing this.reloadRecentVideos(); } else { // if download only mode, just download the file. no redirect - if (forceView === false && this.downloadOnlyMode && !this.iOS) { + if (force_view === false && this.downloadOnlyMode && !this.iOS) { if (is_playlist) { - const zipName = name[0].split(' ')[0] + name[1].split(' ')[0]; - this.downloadPlaylist(name, 'audio', zipName); + this.downloadPlaylist(container['uid']); } else { - this.downloadAudioFile(decodeURI(name)); + this.downloadFileFromServer(container, type); } this.reloadRecentVideos(); } else { localStorage.setItem('player_navigator', this.router.url.split(';')[0]); if (is_playlist) { - this.router.navigate(['/player', {fileNames: name.join('|nvr|'), type: 'audio'}]); + this.router.navigate(['/player', {playlist_id: container['id'], type: type}]); } else { - this.router.navigate(['/player', {type: 'audio', uid: uid}]); - } - } - } - - // remove download from current downloads - this.removeDownloadFromCurrentDownloads(new_download); - } - - downloadHelperMp4(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { - this.downloadingfile = false; - if (this.multiDownloadMode && !this.downloadOnlyMode && !navigate_mode) { - // do nothing - this.reloadRecentVideos(); - } else { - // if download only mode, just download the file. no redirect - if (forceView === false && this.downloadOnlyMode) { - if (is_playlist) { - const zipName = name[0].split(' ')[0] + name[1].split(' ')[0]; - this.downloadPlaylist(name, 'video', zipName); - } else { - this.downloadVideoFile(decodeURI(name)); - } - this.reloadRecentVideos(); - } else { - localStorage.setItem('player_navigator', this.router.url.split(';')[0]); - if (is_playlist) { - this.router.navigate(['/player', {fileNames: name.join('|nvr|'), type: 'video'}]); - } else { - this.router.navigate(['/player', {type: 'video', uid: uid}]); + this.router.navigate(['/player', {type: type, uid: container['uid']}]); } } } @@ -436,124 +405,85 @@ export class MainComponent implements OnInit { // download click handler downloadClicked() { - if (this.ValidURL(this.url)) { - this.urlError = false; - this.path = ''; - - // get common args - const customArgs = (this.customArgsEnabled ? this.customArgs : null); - const customOutput = (this.customOutputEnabled ? this.customOutput : null); - const youtubeUsername = (this.youtubeAuthEnabled && this.youtubeUsername ? this.youtubeUsername : null); - const youtubePassword = (this.youtubeAuthEnabled && this.youtubePassword ? this.youtubePassword : null); - - // set advanced inputs - if (this.allowAdvancedDownload) { - if (customArgs) { - localStorage.setItem('customArgs', customArgs); - } - if (customOutput) { - localStorage.setItem('customOutput', customOutput); - } - if (youtubeUsername) { - localStorage.setItem('youtubeUsername', youtubeUsername); - } - } - - if (this.audioOnly) { - // create download object - const new_download: Download = { - uid: uuid(), - type: 'audio', - percent_complete: 0, - url: this.url, - downloading: true, - is_playlist: this.url.includes('playlist'), - error: false - }; - this.downloads.push(new_download); - if (!this.current_download && !this.multiDownloadMode) { this.current_download = new_download }; - this.downloadingfile = true; - - let customQualityConfiguration = null; - if (this.selectedQuality !== '') { - customQualityConfiguration = this.getSelectedAudioFormat(); - } - - this.postsService.makeMP3(this.url, (this.selectedQuality === '' ? null : this.selectedQuality), - customQualityConfiguration, customArgs, customOutput, youtubeUsername, youtubePassword, new_download.uid).subscribe(posts => { - // update download object - new_download.downloading = false; - new_download.percent_complete = 100; - - const is_playlist = !!(posts['file_names']); - this.path = is_playlist ? posts['file_names'] : posts['audiopathEncoded']; - - this.current_download = null; - - if (this.path !== '-1') { - this.downloadHelperMp3(this.path, posts['uid'], is_playlist, false, new_download); - } - }, error => { // can't access server or failed to download for other reasons - this.downloadingfile = false; - this.current_download = null; - new_download['downloading'] = false; - // removes download from list of downloads - const downloads_index = this.downloads.indexOf(new_download); - if (downloads_index !== -1) { - this.downloads.splice(downloads_index) - } - this.openSnackBar('Download failed!', 'OK.'); - }); - } else { - // create download object - const new_download: Download = { - uid: uuid(), - type: 'video', - percent_complete: 0, - url: this.url, - downloading: true, - is_playlist: this.url.includes('playlist'), - error: false - }; - this.downloads.push(new_download); - if (!this.current_download && !this.multiDownloadMode) { this.current_download = new_download }; - this.downloadingfile = true; - - const customQualityConfiguration = this.getSelectedVideoFormat(); - - this.postsService.makeMP4(this.url, (this.selectedQuality === '' ? null : this.selectedQuality), - customQualityConfiguration, customArgs, customOutput, youtubeUsername, youtubePassword, new_download.uid).subscribe(posts => { - // update download object - new_download.downloading = false; - new_download.percent_complete = 100; - - const is_playlist = !!(posts['file_names']); - this.path = is_playlist ? posts['file_names'] : posts['videopathEncoded']; - - this.current_download = null; - - if (this.path !== '-1') { - this.downloadHelperMp4(this.path, posts['uid'], is_playlist, false, new_download); - } - }, error => { // can't access server - this.downloadingfile = false; - this.current_download = null; - new_download['downloading'] = false; - // removes download from list of downloads - const downloads_index = this.downloads.indexOf(new_download); - if (downloads_index !== -1) { - this.downloads.splice(downloads_index) - } - this.openSnackBar('Download failed!', 'OK.'); - }); - } - - if (this.multiDownloadMode) { - this.url = ''; - this.downloadingfile = false; - } - } else { + if (!this.ValidURL(this.url)) { this.urlError = true; + return; + } + + this.urlError = false; + + // get common args + const customArgs = (this.customArgsEnabled ? this.customArgs : null); + const customOutput = (this.customOutputEnabled ? this.customOutput : null); + const youtubeUsername = (this.youtubeAuthEnabled && this.youtubeUsername ? this.youtubeUsername : null); + const youtubePassword = (this.youtubeAuthEnabled && this.youtubePassword ? this.youtubePassword : null); + + // set advanced inputs + if (this.allowAdvancedDownload) { + if (customArgs) { + localStorage.setItem('customArgs', customArgs); + } + if (customOutput) { + localStorage.setItem('customOutput', customOutput); + } + if (youtubeUsername) { + localStorage.setItem('youtubeUsername', youtubeUsername); + } + } + + const type = this.audioOnly ? 'audio' : 'video'; + // create download object + const new_download: Download = { + uid: uuid(), + type: type, + percent_complete: 0, + url: this.url, + downloading: true, + is_playlist: this.url.includes('playlist'), + error: false + }; + this.downloads.push(new_download); + if (!this.current_download && !this.multiDownloadMode) { this.current_download = new_download }; + this.downloadingfile = true; + + let customQualityConfiguration = type === 'audio' ? this.getSelectedAudioFormat() : this.getSelectedVideoFormat(); + + let cropFileSettings = null; + + if (this.cropFile) { + cropFileSettings = { + cropFileStart: this.cropFileStart, + cropFileEnd: this.cropFileEnd + } + } + + this.postsService.downloadFile(this.url, type, (this.selectedQuality === '' ? null : this.selectedQuality), + customQualityConfiguration, customArgs, customOutput, youtubeUsername, youtubePassword, new_download.uid, cropFileSettings).subscribe(res => { + // update download object + new_download.downloading = false; + new_download.percent_complete = 100; + + const container = res['container']; + const is_playlist = res['file_uids'].length > 1; + + this.current_download = null; + + this.downloadHelper(container, type, is_playlist, false, new_download); + }, error => { // can't access server + this.downloadingfile = false; + this.current_download = null; + new_download['downloading'] = false; + // removes download from list of downloads + const downloads_index = this.downloads.indexOf(new_download); + if (downloads_index !== -1) { + this.downloads.splice(downloads_index) + } + this.openSnackBar('Download failed!', 'OK.'); + }); + + if (this.multiDownloadMode) { + this.url = ''; + this.downloadingfile = false; } } @@ -570,23 +500,26 @@ export class MainComponent implements OnInit { } getSelectedAudioFormat() { - if (this.selectedQuality === '') { return null }; + if (this.selectedQuality === '') { return null; } const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats']; if (cachedFormatsExists) { const audio_formats = this.cachedAvailableFormats[this.url]['formats']['audio']; - return audio_formats[this.selectedQuality]['format_id']; + return this.selectedQuality['format_id']; } else { return null; } } getSelectedVideoFormat() { - if (this.selectedQuality === '') { return null }; - const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats']; - if (cachedFormatsExists) { - const video_formats = this.cachedAvailableFormats[this.url]['formats']['video']; - if (video_formats['best_audio_format'] && this.selectedQuality !== '') { - return video_formats[this.selectedQuality]['format_id'] + '+' + video_formats['best_audio_format']; + if (this.selectedQuality === '') { return null; } + const cachedFormats = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats']; + if (cachedFormats) { + const video_formats = cachedFormats['video']; + if (this.selectedQuality) { + let selected_video_format = this.selectedQuality['format_id']; + // add in audio format if necessary + if (!this.selectedQuality['acodec'] && cachedFormats['best_audio_format']) selected_video_format += `+${cachedFormats['best_audio_format']}`; + return selected_video_format; } } return null; @@ -614,41 +547,27 @@ export class MainComponent implements OnInit { } } - downloadAudioFile(name) { - this.downloading_content['audio'][name] = true; - this.postsService.downloadFileFromServer(name, 'audio').subscribe(res => { - this.downloading_content['audio'][name] = false; + downloadFileFromServer(file, type) { + const ext = type === 'audio' ? 'mp3' : 'mp4' + this.downloading_content[type][file.id] = true; + this.postsService.downloadFileFromServer(file.uid).subscribe(res => { + this.downloading_content[type][file.id] = false; const blob: Blob = res; - saveAs(blob, decodeURIComponent(name) + '.mp3'); + saveAs(blob, decodeURIComponent(file.id) + `.${ext}`); if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded - this.postsService.deleteFile(name, 'video').subscribe(delRes => { + this.postsService.deleteFile(file.uid).subscribe(delRes => { }); } }); } - downloadVideoFile(name) { - this.downloading_content['video'][name] = true; - this.postsService.downloadFileFromServer(name, 'video').subscribe(res => { - this.downloading_content['video'][name] = false; + downloadPlaylist(playlist) { + this.postsService.downloadPlaylistFromServer(playlist.id).subscribe(res => { + if (playlist.id) { this.downloading_content[playlist.type][playlist.id] = false }; const blob: Blob = res; - saveAs(blob, decodeURIComponent(name) + '.mp4'); - - if (!this.fileManagerEnabled) { - // tell server to delete the file once downloaded - this.postsService.deleteFile(name, 'audio').subscribe(delRes => { - }); - } - }); - } - - downloadPlaylist(fileNames, type, zipName = null, playlistID = null) { - this.postsService.downloadFileFromServer(fileNames, type, zipName).subscribe(res => { - if (playlistID) { this.downloading_content[type][playlistID] = false }; - const blob: Blob = res; - saveAs(blob, zipName + '.zip'); + saveAs(blob, playlist.name + '.zip'); }); } @@ -728,9 +647,8 @@ export class MainComponent implements OnInit { this.errorFormats(url); return; } - const parsed_infos = this.getAudioAndVideoFormats(infos.formats); - const available_formats = {audio: parsed_infos[0], video: parsed_infos[1]}; - this.cachedAvailableFormats[url]['formats'] = available_formats; + this.cachedAvailableFormats[url]['formats'] = this.getAudioAndVideoFormats(infos.formats); + console.log(this.cachedAvailableFormats[url]['formats']); }, err => { this.errorFormats(url); }); @@ -773,7 +691,7 @@ export class MainComponent implements OnInit { if (audio_format) { format_array.push('-f', audio_format); } else if (this.selectedQuality) { - format_array.push('--audio-quality', this.selectedQuality); + format_array.push('--audio-quality', this.selectedQuality['format_id']); } // pushes formats @@ -789,7 +707,7 @@ export class MainComponent implements OnInit { if (video_format) { format_array = ['-f', video_format]; } else if (this.selectedQuality) { - format_array = [`bestvideo[height=${this.selectedQuality}]+bestaudio/best[height=${this.selectedQuality}]`]; + format_array = [`bestvideo[height=${this.selectedQuality['format_id']}]+bestaudio/best[height=${this.selectedQuality}]`]; } // pushes formats @@ -886,9 +804,11 @@ export class MainComponent implements OnInit { } } - getAudioAndVideoFormats(formats): any[] { - const audio_formats = {}; - const video_formats = {}; + getAudioAndVideoFormats(formats) { + const audio_formats: any = {}; + const video_formats: any = {}; + + console.log(formats); for (let i = 0; i < formats.length; i++) { const format_obj = {type: null}; @@ -899,9 +819,12 @@ export class MainComponent implements OnInit { format_obj.type = format_type; if (format_obj.type === 'audio' && format.abr) { const key = format.abr.toString() + 'K'; + format_obj['key'] = key; format_obj['bitrate'] = format.abr; format_obj['format_id'] = format.format_id; format_obj['ext'] = format.ext; + format_obj['label'] = key; + // don't overwrite if not m4a if (audio_formats[key]) { if (format.ext === 'm4a') { @@ -912,11 +835,14 @@ export class MainComponent implements OnInit { } } else if (format_obj.type === 'video') { // check if video format is mp4 - const key = format.format_note.replace('p', ''); + const key = `${format.height}p${Math.round(format.fps)}`; if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') { + format_obj['key'] = key; format_obj['height'] = format.height; format_obj['acodec'] = format.acodec; format_obj['format_id'] = format.format_id; + format_obj['label'] = key; + format_obj['fps'] = Math.round(format.fps); // no acodec means no overwrite if (!(video_formats[key]) || format_obj['acodec'] !== 'none') { @@ -926,9 +852,17 @@ export class MainComponent implements OnInit { } } - video_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats); + const parsed_formats: any = {}; - return [audio_formats, video_formats] + parsed_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats); + + parsed_formats['video'] = Object.values(video_formats); + parsed_formats['audio'] = Object.values(audio_formats); + + parsed_formats['video'] = parsed_formats['video'].sort((a, b) => b.height - a.height || b.fps - a.fps); + parsed_formats['audio'] = parsed_formats['audio'].sort((a, b) => b.bitrate - a.bitrate); + + return parsed_formats; } getBestAudioFormatForMp4(audio_formats) { diff --git a/src/app/player/player.component.css b/src/app/player/player.component.css index 202b80d..dcd9d4c 100644 --- a/src/app/player/player.component.css +++ b/src/app/player/player.component.css @@ -9,7 +9,6 @@ .audio-styles { height: 50px; - background-color: transparent; width: 100%; } diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 086adf6..a0dd7d2 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -1,14 +1,14 @@
-
+
-
- -