From c6fc5352c5588e0617ff980b459cc99b4a7811d5 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Wed, 9 Dec 2020 17:28:00 -0500 Subject: [PATCH 01/23] Added ability to add more metadata to db through migrations, and added scaffolding for supporting description and play count in the player component --- backend/app.js | 32 ++++++++++++++++++++++++++++ backend/db.js | 9 +++++++- backend/utils.js | 7 +++++- src/app/player/player.component.html | 12 +++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/backend/app.js b/backend/app.js index 0cdd414..148e94c 100644 --- a/backend/app.js +++ b/backend/app.js @@ -216,6 +216,16 @@ async function checkMigrations() { else { logger.error('Migration failed: 3.5->3.6+'); } } + // 4.1->4.2 migration + + const add_description_migration_complete = false; // db.get('add_description_migration_complete').value(); + if (!add_description_migration_complete) { + logger.info('Beginning migration: 4.1->4.2+') + const success = await addMetadataPropertyToDB('description'); + if (success) { logger.info('4.1->4.2+ migration complete!'); } + else { logger.error('Migration failed: 4.1->4.2+'); } + } + return true; } @@ -251,6 +261,28 @@ async function runFilesToDBMigration() { } } +async function addMetadataPropertyToDB(property_key) { + try { + const dirs_to_check = db_api.getFileDirectoriesAndDBs(); + for (const dir_to_check of dirs_to_check) { + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); + for (const file of files) { + if (file[property_key]) { + dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); + } + } + } + + // sets migration to complete + db.set('add_description_migration_complete', true).write(); + return true; + } catch(err) { + logger.error(err); + return false; + } +} + async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku diff --git a/backend/db.js b/backend/db.js index a2ddb7a..9436715 100644 --- a/backend/db.js +++ b/backend/db.js @@ -115,7 +115,7 @@ function getAppendedBasePathSub(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } -async function importUnregisteredFiles() { +function getFileDirectoriesAndDBs() { let dirs_to_check = []; let subscriptions_to_check = []; const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode @@ -181,6 +181,12 @@ async function importUnregisteredFiles() { }); } + return dirs_to_check; +} + +async function importUnregisteredFiles() { + const dirs_to_check = getFileDirectoriesAndDBs(); + // run through check list and check each file to see if it's missing from the db for (const dir_to_check of dirs_to_check) { // recursively get all files in dir's path @@ -203,5 +209,6 @@ module.exports = { initialize: initialize, registerFileDB: registerFileDB, updatePlaylist: updatePlaylist, + getFileDirectoriesAndDBs: getFileDirectoriesAndDBs, importUnregisteredFiles: importUnregisteredFiles } diff --git a/backend/utils.js b/backend/utils.js index ed3b584..33b877a 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -20,7 +20,7 @@ function getTrueFileName(unfixed_path, type) { return fixed_path; } -async function getDownloadedFilesByType(basePath, type) { +async function getDownloadedFilesByType(basePath, type, full_metadata = false) { // return empty array if the path doesn't exist if (!(await fs.pathExists(basePath))) return []; @@ -36,6 +36,11 @@ async function getDownloadedFilesByType(basePath, type) { var id = file_path.substring(0, file_path.length-4); var jsonobj = await getJSONByType(type, id, basePath); if (!jsonobj) continue; + if (full_metadata) { + jsonobj['id'] = id; + files.push(jsonobj); + continue; + } var title = jsonobj.title; var url = jsonobj.webpage_url; var uploader = jsonobj.uploader; diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 6774ace..73ea9eb 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -8,6 +8,18 @@ +
+
+
+
+ {{file_obj['local_play_count'] ? file_obj['local_play_count'] : 0}} views +
+
+ +
+
+
+
{{playlist_item.label}} From 4f693d4eda5d28230d1112d96def19cda198abfc Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 14 Dec 2020 18:19:50 -0500 Subject: [PATCH 02/23] Added description to player component and simplified the database by un-splitting videos and playlists by type --- backend/app.js | 150 +++++++++--------- backend/authentication/auth.js | 66 +++----- backend/db.js | 20 +-- backend/utils.js | 3 +- src/app/app.module.ts | 8 +- .../see-more/see-more.component.html | 11 ++ .../see-more/see-more.component.scss | 7 + .../see-more/see-more.component.spec.ts | 25 +++ .../components/see-more/see-more.component.ts | 60 +++++++ src/app/player/player.component.html | 30 ++-- 10 files changed, 235 insertions(+), 145 deletions(-) create mode 100644 src/app/components/see-more/see-more.component.html create mode 100644 src/app/components/see-more/see-more.component.scss create mode 100644 src/app/components/see-more/see-more.component.spec.ts create mode 100644 src/app/components/see-more/see-more.component.ts diff --git a/backend/app.js b/backend/app.js index 148e94c..b2d6446 100644 --- a/backend/app.js +++ b/backend/app.js @@ -13,7 +13,7 @@ var express = require("express"); var bodyParser = require("body-parser"); var archiver = require('archiver'); var unzipper = require('unzipper'); -var db_api = require('./db') +var db_api = require('./db'); var utils = require('./utils') var mergeFiles = require('merge-files'); const low = require('lowdb') @@ -87,14 +87,8 @@ categories_api.initialize(db, users_db, logger, db_api); // Set some defaults db.defaults( { - playlists: { - audio: [], - video: [] - }, - files: { - audio: [], - video: [] - }, + playlists: [], + files: [], configWriteFlag: false, downloads: {}, subscriptions: [], @@ -218,10 +212,12 @@ async function checkMigrations() { // 4.1->4.2 migration - const add_description_migration_complete = false; // db.get('add_description_migration_complete').value(); + const add_description_migration_complete = db.get('add_description_migration_complete').value(); if (!add_description_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') - const success = await addMetadataPropertyToDB('description'); + let success = await simplifyDBFileStructure(); + success = success && await addMetadataPropertyToDB('view_count'); + success = success && await addMetadataPropertyToDB('description'); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } @@ -229,6 +225,7 @@ async function checkMigrations() { return true; } +/* async function runFilesToDBMigration() { try { let mp3s = await getMp3s(); @@ -260,6 +257,37 @@ async function runFilesToDBMigration() { return false; } } +*/ + +async function simplifyDBFileStructure() { + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + const user = users[i]; + if (user['files']['video'] !== undefined && user['files']['audio'] !== undefined) { + const user_files = user['files']['video'].concat(user['files']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({files: user_files}).write(); + } + if (user['playlists']['video'] !== undefined && user['playlists']['audio'] !== undefined) { + const user_playlists = user['playlists']['video'].concat(user['playlists']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({playlists: user_playlists}).write(); + } + } + + if (db.get('files.video').value() !== undefined && db.get('files.audio').value() !== undefined) { + const files = db.get('files.video').value().concat(db.get('files.audio')); + db.assign({files: files}).write(); + } + + if (db.get('playlists.video').value() !== undefined && db.get('playlists.audio').value() !== undefined) { + const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio')); + db.assign({playlists: playlists}).write(); + } + + + return true; +} async function addMetadataPropertyToDB(property_key) { try { @@ -592,6 +620,9 @@ async function loadConfig() { // creates archive path if missing await fs.ensureDir(archivePath); + // check migrations + await checkMigrations(); + // now this is done here due to youtube-dl's repo takedown await startYoutubeDL(); @@ -606,9 +637,6 @@ async function loadConfig() { db_api.importUnregisteredFiles(); - // check migrations - await checkMigrations(); - // load in previous downloads downloads = db.get('downloads').value(); @@ -1994,7 +2022,7 @@ async function addThumbnails(files) { // gets all download mp3s app.get('/api/getMp3s', optionalJwt, async function(req, res) { - var mp3s = db.get('files.audio').value(); // getMp3s(); + var mp3s = db.get('files').chain().find({isAudio: true}).value(); // getMp3s(); var playlists = db.get('playlists.audio').value(); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { @@ -2020,8 +2048,8 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { // gets all download mp4s app.get('/api/getMp4s', optionalJwt, async function(req, res) { - var mp4s = db.get('files.video').value(); // getMp4s(); - var playlists = db.get('playlists.video').value(); + var mp4s = db.get('files').chain().find({isAudio: false}).value(); // getMp4s(); + var playlists = db.get('playlists').value(); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { @@ -2052,21 +2080,11 @@ app.post('/api/getFile', optionalJwt, function (req, res) { var file = null; if (req.isAuthenticated()) { - file = auth_api.getUserVideo(req.user.uid, uid, type); + file = auth_api.getUserVideo(req.user.uid, uid); } else if (uuid) { - file = auth_api.getUserVideo(uuid, uid, type, true); + file = auth_api.getUserVideo(uuid, uid, true); } else { - if (!type) { - file = db.get('files.audio').find({uid: uid}).value(); - if (!file) { - file = db.get('files.video').find({uid: uid}).value(); - if (file) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!file && type) file = db.get(`files.${type}`).find({uid: uid}).value(); + file = db.get('files').find({uid: uid}).value(); } // check if chat exists for twitch videos @@ -2086,32 +2104,20 @@ app.post('/api/getFile', optionalJwt, function (req, res) { app.post('/api/getAllFiles', optionalJwt, async function (req, res) { // these are returned - let files = []; - let playlists = []; - let subscription_files = []; + let files = null; + let playlists = null; - let videos = null; - let audios = null; - let audio_playlists = null; - let video_playlists = null; let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getAllSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; // get basic info depending on multi-user mode being enabled if (req.isAuthenticated()) { - videos = auth_api.getUserVideos(req.user.uid, 'video'); - audios = auth_api.getUserVideos(req.user.uid, 'audio'); - audio_playlists = auth_api.getUserPlaylists(req.user.uid, 'audio'); - video_playlists = auth_api.getUserPlaylists(req.user.uid, 'video'); + files = auth_api.getUserVideos(req.user.uid); + playlists = auth_api.getUserPlaylists(req.user.uid); } else { - videos = db.get('files.audio').value(); - audios = db.get('files.video').value(); - audio_playlists = db.get('playlists.audio').value(); - video_playlists = db.get('playlists.video').value(); + files = db.get('files').value(); + playlists = db.get('playlists').value(); } - files = videos.concat(audios); - playlists = video_playlists.concat(audio_playlists); - // loop through subscriptions and add videos for (let i = 0; i < subscriptions.length; i++) { sub = subscriptions[i]; @@ -2187,14 +2193,13 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { // video sharing app.post('/api/enableSharing', optionalJwt, function(req, res) { - var type = req.body.type; var uid = req.body.uid; var is_playlist = req.body.is_playlist; let success = false; // multi-user mode if (req.isAuthenticated()) { // if multi user mode, use this method instead - success = auth_api.changeSharingMode(req.user.uid, uid, type, is_playlist, true); + success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, true); res.send({success: success}); return; } @@ -2203,12 +2208,12 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files.${type}`) + db.get(`files`) .find({uid: uid}) .assign({sharingEnabled: true}) .write(); } else if (is_playlist) { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: uid}) .assign({sharingEnabled: true}) .write(); @@ -2237,7 +2242,7 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { // multi-user mode if (req.isAuthenticated()) { // if multi user mode, use this method instead - success = auth_api.changeSharingMode(req.user.uid, uid, type, is_playlist, false); + success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, false); res.send({success: success}); return; } @@ -2246,12 +2251,12 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files.${type}`) + db.get(`files`) .find({uid: uid}) .assign({sharingEnabled: false}) .write(); } else if (is_playlist) { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: uid}) .assign({sharingEnabled: false}) .write(); @@ -2544,7 +2549,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { if (req.isAuthenticated()) { auth_api.addPlaylist(req.user.uid, new_playlist, type); } else { - db.get(`playlists.${type}`) + db.get(`playlists`) .push(new_playlist) .write(); } @@ -2558,26 +2563,15 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { app.post('/api/getPlaylist', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; - let type = req.body.type; let uuid = req.body.uuid; let playlist = null; if (req.isAuthenticated()) { - playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID, type); + playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID); type = playlist.type; } else { - if (!type) { - playlist = db.get('playlists.audio').find({id: playlistID}).value(); - if (!playlist) { - playlist = db.get('playlists.video').find({id: playlistID}).value(); - if (playlist) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!playlist) playlist = db.get(`playlists.${type}`).find({id: playlistID}).value(); + playlist = db.get(`playlists`).find({id: playlistID}).value(); } res.send({ @@ -2590,14 +2584,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; let fileNames = req.body.fileNames; - let type = req.body.type; let success = false; try { if (req.isAuthenticated()) { - auth_api.updatePlaylistFiles(req.user.uid, playlistID, fileNames, type); + auth_api.updatePlaylistFiles(req.user.uid, playlistID, fileNames); } else { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: playlistID}) .assign({fileNames: fileNames}) .write(); @@ -2623,15 +2616,14 @@ app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; - let type = req.body.type; let success = null; try { if (req.isAuthenticated()) { - auth_api.removePlaylist(req.user.uid, playlistID, type); + auth_api.removePlaylist(req.user.uid, playlistID); } else { // removes playlist from playlists - db.get(`playlists.${type}`) + db.get(`playlists`) .remove({id: playlistID}) .write(); } @@ -2653,23 +2645,23 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { var blacklistMode = req.body.blacklistMode; if (req.isAuthenticated()) { - let success = auth_api.deleteUserFile(req.user.uid, uid, type, blacklistMode); + let success = auth_api.deleteUserFile(req.user.uid, uid, blacklistMode); res.send(success); return; } - var file_obj = db.get(`files.${type}`).find({uid: uid}).value(); + var file_obj = db.get(`files`).find({uid: uid}).value(); var name = file_obj.id; var fullpath = file_obj ? file_obj.path : null; var wasDeleted = false; if (await fs.pathExists(fullpath)) { wasDeleted = type === 'audio' ? await deleteAudioFile(name, path.basename(fullpath), blacklistMode) : await deleteVideoFile(name, path.basename(fullpath), blacklistMode); - db.get('files.video').remove({uid: uid}).write(); + db.get('files').remove({uid: uid}).write(); // wasDeleted = true; res.send(wasDeleted); } else if (video_obj) { - db.get('files.video').remove({uid: uid}).write(); + db.get('files').remove({uid: uid}).write(); wasDeleted = true; res.send(wasDeleted); } else { diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 9fdc7e7..3c77edd 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -281,24 +281,13 @@ exports.adminExists = function() { // video stuff -exports.getUserVideos = function(user_uid, type) { +exports.getUserVideos = function(user_uid) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['files'][type]; + return user['files']; } -exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false) { - let file = null; - if (!type) { - file = users_db.get('users').find({uid: user_uid}).get(`files.audio`).find({uid: file_uid}).value(); - if (!file) { - file = users_db.get('users').find({uid: user_uid}).get(`files.video`).find({uid: file_uid}).value(); - if (file) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!file && type) file = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); +exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { + let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); // prevent unauthorized users from accessing the file info if (file && !file['sharingEnabled'] && requireSharing) file = null; @@ -306,38 +295,28 @@ exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false return file; } -exports.addPlaylist = function(user_uid, new_playlist, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).push(new_playlist).write(); +exports.addPlaylist = function(user_uid, new_playlist) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write(); return true; } -exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).assign({fileNames: new_filenames}); +exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames}); return true; } -exports.removePlaylist = function(user_uid, playlistID, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).remove({id: playlistID}).write(); +exports.removePlaylist = function(user_uid, playlistID) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write(); return true; } -exports.getUserPlaylists = function(user_uid, type) { +exports.getUserPlaylists = function(user_uid) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['playlists'][type]; + return user['playlists']; } -exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing = false) { - let playlist = null; - if (!type) { - playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.audio`).find({id: playlistID}).value(); - if (!playlist) { - playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.video`).find({id: playlistID}).value(); - if (playlist) type = 'video'; - } else { - type = 'audio'; - } - } - if (!playlist) playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).value(); +exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { + let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value(); // prevent unauthorized users from accessing the file info if (requireSharing && !playlist['sharingEnabled']) playlist = null; @@ -345,21 +324,22 @@ exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing = return playlist; } -exports.registerUserFile = function(user_uid, file_object, type) { - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) +exports.registerUserFile = function(user_uid, file_object) { + users_db.get('users').find({uid: user_uid}).get(`files`) .remove({ path: file_object['path'] }).write(); - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) + users_db.get('users').find({uid: user_uid}).get(`files`) .push(file_object) .write(); } -exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) { +exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) { let success = false; - const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); + const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); if (file_obj) { + const type = file_obj.isAudio ? 'audio' : 'video'; const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const ext = type === 'audio' ? '.mp3' : '.mp4'; @@ -375,7 +355,7 @@ exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode } const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext); - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) + users_db.get('users').find({uid: user_uid}).get(`files`) .remove({ uid: file_uid }).write(); @@ -424,11 +404,11 @@ exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode return success; } -exports.changeSharingMode = function(user_uid, file_uid, type, is_playlist, enabled) { +exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) { let success = false; const user_db_obj = users_db.get('users').find({uid: user_uid}); if (user_db_obj.value()) { - const file_db_obj = is_playlist ? user_db_obj.get(`playlists.${type}`).find({id: file_uid}) : user_db_obj.get(`files.${type}`).find({uid: file_uid}); + const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid}); if (file_db_obj.value()) { success = true; file_db_obj.assign({sharingEnabled: enabled}).write(); diff --git a/backend/db.js b/backend/db.js index 9436715..f95b43f 100644 --- a/backend/db.js +++ b/backend/db.js @@ -32,9 +32,9 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null, custo if (!sub) { if (multiUserMode) { const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get(`files.${type}`); + db_path = users_db.get('users').find({uid: user_uid}).get(`files`); } else { - db_path = db.get(`files.${type}`) + db_path = db.get(`files`); } } else { if (multiUserMode) { @@ -94,18 +94,18 @@ function generateFileObject(id, type, customPath = null, sub = null) { var thumbnail = jsonobj.thumbnail; var duration = jsonobj.duration; var isaudio = type === 'audio'; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); + var description = jsonobj.description; + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description); return file_obj; } function updatePlaylist(playlist, user_uid) { let playlistID = playlist.id; - let type = playlist.type; let db_loc = null; if (user_uid) { - db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}); + db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}); } else { - db_loc = db.get(`playlists.${type}`).find({id: playlistID}); + db_loc = db.get(`playlists`).find({id: playlistID}); } db_loc.assign(playlist).write(); return true; @@ -132,14 +132,14 @@ function getFileDirectoriesAndDBs() { // add user's audio dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'audio'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files.audio'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'audio' }); // add user's video dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'video'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files.video'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'video' }); } @@ -153,14 +153,14 @@ function getFileDirectoriesAndDBs() { // add audio dir to check list dirs_to_check.push({ basePath: audioFolderPath, - dbPath: db.get('files.audio'), + dbPath: db.get('files'), type: 'audio' }); // add video dir to check list dirs_to_check.push({ basePath: videoFolderPath, - dbPath: db.get('files.video'), + dbPath: db.get('files'), type: 'video' }); } diff --git a/backend/utils.js b/backend/utils.js index 33b877a..988a112 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -189,7 +189,7 @@ async function recFindByExt(base,ext,files,result) // objects -function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) { +function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description) { this.id = id; this.title = title; this.thumbnailURL = thumbnailURL; @@ -200,6 +200,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p this.size = size; this.path = path; this.upload_date = upload_date; + this.description = description; } module.exports = { diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 72c2658..3f9a119 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -81,6 +81,7 @@ import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dia import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component'; import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component'; +import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.component'; registerLocaleData(es, 'es'); @@ -107,6 +108,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible VideoInfoDialogComponent, ArgModifierDialogComponent, HighlightPipe, + LinkifyPipe, UpdaterComponent, UpdateProgressDialogComponent, ShareMediaDialogComponent, @@ -127,7 +129,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible EditSubscriptionDialogComponent, CustomPlaylistsComponent, EditCategoryDialogComponent, - TwitchChatComponent + TwitchChatComponent, + SeeMoreComponent ], imports: [ CommonModule, @@ -188,7 +191,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible PostsService ], exports: [ - HighlightPipe + HighlightPipe, + LinkifyPipe ], bootstrap: [AppComponent] }) diff --git a/src/app/components/see-more/see-more.component.html b/src/app/components/see-more/see-more.component.html new file mode 100644 index 0000000..297d820 --- /dev/null +++ b/src/app/components/see-more/see-more.component.html @@ -0,0 +1,11 @@ + + + + + See more. + + + See less. + + + \ No newline at end of file diff --git a/src/app/components/see-more/see-more.component.scss b/src/app/components/see-more/see-more.component.scss new file mode 100644 index 0000000..1843c11 --- /dev/null +++ b/src/app/components/see-more/see-more.component.scss @@ -0,0 +1,7 @@ +.text { + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + word-wrap: break-word; + } \ No newline at end of file diff --git a/src/app/components/see-more/see-more.component.spec.ts b/src/app/components/see-more/see-more.component.spec.ts new file mode 100644 index 0000000..608bee6 --- /dev/null +++ b/src/app/components/see-more/see-more.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { SeeMoreComponent } from './see-more.component'; + +describe('SeeMoreComponent', () => { + let component: SeeMoreComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ SeeMoreComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(SeeMoreComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/see-more/see-more.component.ts b/src/app/components/see-more/see-more.component.ts new file mode 100644 index 0000000..8573cb9 --- /dev/null +++ b/src/app/components/see-more/see-more.component.ts @@ -0,0 +1,60 @@ +import { Component, Input, OnInit, Pipe, PipeTransform } from '@angular/core'; +import { DomSanitizer } from '@angular/platform-browser'; + +@Pipe({ name: 'linkify' }) +export class LinkifyPipe implements PipeTransform { + + constructor(private _domSanitizer: DomSanitizer) {} + + transform(value: any, args?: any): any { + return this._domSanitizer.bypassSecurityTrustHtml(this.stylize(value)); + } + + // Modify this method according to your custom logic + private stylize(text: string): string { + let stylizedText: string = ''; + if (text && text.length > 0) { + for (let line of text.split("\n")) { + for (let t of line.split(" ")) { + if (t.startsWith("http") && t.length>7) { + stylizedText += `${t} `; + } + else + stylizedText += t + " "; + } + stylizedText += '
'; + } + return stylizedText; + } + else return text; + } + +} + +@Component({ + selector: 'app-see-more', + templateUrl: './see-more.component.html', + providers: [LinkifyPipe], + styleUrls: ['./see-more.component.scss'] +}) +export class SeeMoreComponent implements OnInit { + + see_more_active = false; + + @Input() text = ''; + @Input() line_limit = 2; + + constructor() { } + + ngOnInit(): void { + } + + toggleSeeMore() { + this.see_more_active = !this.see_more_active; + } + + parseText() { + return this.text.replace(/(http.*?\s)/, "$1") + } + +} diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 73ea9eb..a328561 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -8,14 +8,28 @@
-
+
-
- {{file_obj['local_play_count'] ? file_obj['local_play_count'] : 0}} views +
+ {{db_file['local_view_count'] ? db_file['local_view_count'] : 0}} views
-
- +
+ +

+ +

+ + +

+ No description available. +

+ +
+
+
+ +
@@ -25,15 +39,11 @@ {{playlist_item.label}}
- + - -
- -
-
+
Unit is seconds, only include numbers.
+
+ Redownload fresh uploads +
From 59c38321fd938c3d3a609c4895c7317f0c157628 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 01:46:19 -0500 Subject: [PATCH 07/23] Fixed bug in file deletion --- backend/app.js | 4 ++-- .../components/recent-videos/recent-videos.component.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/app.js b/backend/app.js index 6476136..d3f9d57 100644 --- a/backend/app.js +++ b/backend/app.js @@ -2570,7 +2570,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { var blacklistMode = req.body.blacklistMode; if (req.isAuthenticated()) { - let success = auth_api.deleteUserFile(req.user.uid, uid, blacklistMode); + let success = await auth_api.deleteUserFile(req.user.uid, uid, blacklistMode); res.send(success); return; } @@ -2583,7 +2583,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { { wasDeleted = type === 'audio' ? await deleteAudioFile(name, path.basename(fullpath), blacklistMode) : await deleteVideoFile(name, path.basename(fullpath), blacklistMode); db.get('files').remove({uid: uid}).write(); - // wasDeleted = true; + wasDeleted = true; res.send(wasDeleted); } else if (video_obj) { db.get('files').remove({uid: uid}).write(); diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index a0667c7..e745138 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -127,9 +127,11 @@ export class RecentVideosComponent implements OnInit { this.normal_files_received = false; this.postsService.getAllFiles().subscribe(res => { this.files = res['files']; - this.files.forEach(file => { + for (let i = 0; i < this.files.length; i++) { + const file = this.files[i]; file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration); - }); + file.index = i; + } this.files.sort(this.sortFiles); if (this.search_mode) { this.filterFiles(this.search_text); @@ -247,7 +249,8 @@ export class RecentVideosComponent implements OnInit { this.postsService.deleteFile(file.uid, file.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { if (result) { this.postsService.openSnackBar('Delete success!', 'OK.'); - this.files.splice(index, 1); + this.files.splice(file.index, 1); + this.filterByProperty(this.filterProperty['property']); } else { this.postsService.openSnackBar('Delete failed!', 'OK.'); } From eb7661c14a3e7078f7a43944bec316d80f016659 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 02:06:52 -0500 Subject: [PATCH 08/23] Fixed bug in file deletion where file indexes became stale --- src/app/components/recent-videos/recent-videos.component.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index e745138..2da73f0 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -250,6 +250,7 @@ export class RecentVideosComponent implements OnInit { if (result) { this.postsService.openSnackBar('Delete success!', 'OK.'); this.files.splice(file.index, 1); + for (let i = 0; i < this.files.length; i++) { this.files[i].index = i } this.filterByProperty(this.filterProperty['property']); } else { this.postsService.openSnackBar('Delete failed!', 'OK.'); From 441a470990a7a689897ee0e4a0e10e179293d527 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 03:51:30 -0500 Subject: [PATCH 09/23] Added ability to view playlist in reverse order in the playlist editing dialog --- .../modify-playlist.component.html | 20 ++++++++++++++----- .../modify-playlist.component.scss | 5 ----- .../modify-playlist.component.ts | 15 ++++++++++++++ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.html b/src/app/dialogs/modify-playlist/modify-playlist.component.html index d35db91..69f4cad 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.html +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.html @@ -8,14 +8,24 @@
+
+
+ Normal order  + Reverse order  + +
+ +
+ +
+
+ -
{{playlist_item}}
 + +
{{playlist_item}}
 - -
- -
+ diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.scss b/src/app/dialogs/modify-playlist/modify-playlist.component.scss index 84d3c54..0be9a78 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.scss +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.scss @@ -30,11 +30,6 @@ border: none; transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); } -.add-content-button { -margin-top: 15px; -margin-bottom: 10px; -} - .remove-item-button { right: 10px; position: absolute; diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.ts b/src/app/dialogs/modify-playlist/modify-playlist.component.ts index 18db662..414fc92 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts @@ -15,6 +15,7 @@ export class ModifyPlaylistComponent implements OnInit { available_files = []; all_files = []; playlist_updated = false; + reverse_order = false; constructor(@Inject(MAT_DIALOG_DATA) public data: any, private postsService: PostsService, @@ -26,6 +27,8 @@ export class ModifyPlaylistComponent implements OnInit { this.original_playlist = JSON.parse(JSON.stringify(this.data.playlist)); this.getFiles(); } + + this.reverse_order = localStorage.getItem('default_playlist_order_reversed') === 'true'; } getFiles() { @@ -72,11 +75,23 @@ export class ModifyPlaylistComponent implements OnInit { } removeContent(index) { + if (this.reverse_order) { + index = this.playlist.fileNames.length - 1 - index; + } this.playlist.fileNames.splice(index, 1); this.processFiles(); } + togglePlaylistOrder() { + this.reverse_order = !this.reverse_order; + localStorage.setItem('default_playlist_order_reversed', '' + this.reverse_order); + } + drop(event: CdkDragDrop) { + if (this.reverse_order) { + event.previousIndex = this.playlist.fileNames.length - 1 - event.previousIndex; + event.currentIndex = this.playlist.fileNames.length - 1 - event.currentIndex; + } moveItemInArray(this.playlist.fileNames, event.previousIndex, event.currentIndex); } From e75b56ad3fe1efa9598f6cfe4be5ca50a577cbc0 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 04:12:27 -0500 Subject: [PATCH 10/23] Added ability to pause specific subscriptions --- backend/app.js | 8 +++++--- .../edit-subscription-dialog.component.html | 7 +++++-- .../subscription-info-dialog.component.html | 2 +- .../subscription/subscription/subscription.component.html | 2 +- src/app/subscriptions/subscriptions.component.html | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/app.js b/backend/app.js index d3f9d57..30e19b4 100644 --- a/backend/app.js +++ b/backend/app.js @@ -701,12 +701,14 @@ async function watchSubscriptions() { if (!subscriptions) return; - let subscriptions_amount = subscriptions.length; + const valid_subscriptions = subscriptions.filter(sub => !sub.paused); + + let subscriptions_amount = valid_subscriptions.length; let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount); let current_delay = 0; - for (let i = 0; i < subscriptions.length; i++) { - let sub = subscriptions[i]; + for (let i = 0; i < valid_subscriptions.length; i++) { + let sub = valid_subscriptions[i]; // don't check the sub if the last check for the same subscription has not completed if (subscription_timeouts[sub.id]) { diff --git a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html index 00d5485..f76af9f 100644 --- a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html +++ b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html @@ -1,8 +1,11 @@ -

Editing

 {{sub.name}} +

Editing {{sub.name}} (Paused)

+
+ Paused +
Download all uploads
@@ -31,7 +34,7 @@
-
+
Streaming-only mode
diff --git a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html index 1ebff5a..4517a47 100644 --- a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html +++ b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html @@ -1,4 +1,4 @@ -

{{sub.name}}

+

{{sub.name}} (Paused)

diff --git a/src/app/subscription/subscription/subscription.component.html b/src/app/subscription/subscription/subscription.component.html index 0d8c917..e404df3 100644 --- a/src/app/subscription/subscription/subscription.component.html +++ b/src/app/subscription/subscription/subscription.component.html @@ -2,7 +2,7 @@

- {{subscription.name}} + {{subscription.name}} (Paused)

diff --git a/src/app/subscriptions/subscriptions.component.html b/src/app/subscriptions/subscriptions.component.html index 5bf76bb..1c8e838 100644 --- a/src/app/subscriptions/subscriptions.component.html +++ b/src/app/subscriptions/subscriptions.component.html @@ -9,7 +9,7 @@ - {{ sub.name }} + {{ sub.name }} (Paused)
Name not available. Channel retrieval in progress.
@@ -28,7 +28,7 @@ - {{ sub.name }} + {{ sub.name }} (Paused)
Name not available. Playlist retrieval in progress.
From 7e06d30205ec81deb970af37c09e52430b441d60 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 13:03:49 -0500 Subject: [PATCH 11/23] 401 unauthorized requests now redirect users to the login page --- src/app/app.module.ts | 6 ++-- src/app/components/login/login.component.ts | 2 +- src/app/http.interceptor.ts | 34 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 src/app/http.interceptor.ts diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 6ef4efd..0ae868f 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -32,7 +32,7 @@ import { DragDropModule } from '@angular/cdk/drag-drop'; import { ClipboardModule } from '@angular/cdk/clipboard'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { AppComponent } from './app.component'; -import { HttpClientModule, HttpClient } from '@angular/common/http'; +import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http'; import { PostsService } from 'app/posts.services'; import { FileCardComponent } from './file-card/file-card.component'; import { RouterModule } from '@angular/router'; @@ -85,6 +85,7 @@ import { CustomPlaylistsComponent } from './components/custom-playlists/custom-p import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component'; import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.component'; +import { H401Interceptor } from './http.interceptor'; registerLocaleData(es, 'es'); @@ -191,7 +192,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible SettingsComponent ], providers: [ - PostsService + PostsService, + { provide: HTTP_INTERCEPTORS, useClass: H401Interceptor, multi: true } ], exports: [ HighlightPipe, diff --git a/src/app/components/login/login.component.ts b/src/app/components/login/login.component.ts index d5c5816..edf7f99 100644 --- a/src/app/components/login/login.component.ts +++ b/src/app/components/login/login.component.ts @@ -27,7 +27,7 @@ export class LoginComponent implements OnInit { constructor(private postsService: PostsService, private snackBar: MatSnackBar, private router: Router) { } ngOnInit(): void { - if (this.postsService.isLoggedIn) { + if (this.postsService.isLoggedIn && localStorage.getItem('jwt_token') !== 'null') { this.router.navigate(['/home']); } this.postsService.service_initialized.subscribe(init => { diff --git a/src/app/http.interceptor.ts b/src/app/http.interceptor.ts new file mode 100644 index 0000000..edde22b --- /dev/null +++ b/src/app/http.interceptor.ts @@ -0,0 +1,34 @@ +import { HttpEvent, HttpHandler, HttpInterceptor, HttpRequest } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { MatSnackBar } from '@angular/material/snack-bar'; +import { Router } from '@angular/router'; +import { Observable, throwError } from 'rxjs'; +import { catchError } from 'rxjs/operators'; + +@Injectable() +export class H401Interceptor implements HttpInterceptor { + + constructor(private router: Router, private snackBar: MatSnackBar) { } + + intercept(request: HttpRequest, next: HttpHandler): Observable> { + return next.handle(request).pipe(catchError(err => { + if (err.status === 401) { + localStorage.setItem('jwt_token', null); + if (this.router.url !== '/login') { + this.router.navigate(['/login']).then(() => { + this.openSnackBar('Login expired, please login again.'); + }); + } + } + + const error = err.error.message || err.statusText; + return throwError(error); + })); + } + + public openSnackBar(message: string, action: string = '') { + this.snackBar.open(message, action, { + duration: 2000, + }); + } +} \ No newline at end of file From 4cbfab20e00c6d888daa14441746a4a2c4a9af45 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 19 Dec 2020 14:33:19 -0500 Subject: [PATCH 12/23] Added category info to video info dialog (if present) - you can now search by category on the home page --- src/app/components/recent-videos/recent-videos.component.ts | 2 +- .../video-info-dialog/video-info-dialog.component.html | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index 2da73f0..7ca6c9b 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -98,7 +98,7 @@ export class RecentVideosComponent implements OnInit { private filterFiles(value: string) { const filterValue = value.toLowerCase(); - this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue)); + this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.toLowerCase().includes(filterValue)); this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}); } diff --git a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html index 7323385..203cfd5 100644 --- a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html +++ b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html @@ -25,6 +25,10 @@
Upload Date: 
{{file.upload_date ? file.upload_date : 'N/A'}}
+
+
Category: 
+
{{file.category}}N/A
+
From afb5e3800c675416f657ee718d04b2559307b15f Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 20 Dec 2020 00:30:48 -0500 Subject: [PATCH 13/23] In the subscription page, the subscription is now continuously retrieved at an interval of 1s to update for new videos or the downloading state - There is now a visual indicator for when a subscription is retrieving videos --- Public API v1.yaml | 4 +- backend/app.js | 24 ++++----- backend/subscriptions.js | 50 ++++++++++++++++--- src/app/posts.services.ts | 2 +- .../subscription/subscription.component.html | 1 + .../subscription/subscription.component.ts | 25 ++++------ 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/Public API v1.yaml b/Public API v1.yaml index e82328a..b57c386 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -261,12 +261,12 @@ paths: $ref: '#/components/schemas/inline_response_200_10' security: - Auth query parameter: [] - /api/getAllSubscriptions: + /api/getSubscriptions: post: tags: - subscriptions summary: Get all subscriptions - operationId: post-api-getAllSubscriptions + operationId: post-api-getSubscriptions requestBody: content: application/json: diff --git a/backend/app.js b/backend/app.js index 0f6db8c..e1790fd 100644 --- a/backend/app.js +++ b/backend/app.js @@ -628,6 +628,9 @@ async function loadConfig() { // get subscriptions if (allowSubscriptions) { + // set downloading to false + let subscriptions = subscriptions_api.getAllSubscriptions(); + subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds watchSubscriptions(); setInterval(() => { @@ -686,18 +689,7 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) { } async function watchSubscriptions() { - let subscriptions = null; - - const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode) { - subscriptions = []; - let users = users_db.get('users').value(); - for (let i = 0; i < users.length; i++) { - if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); - } - } else { - subscriptions = subscriptions_api.getAllSubscriptions(); - } + let subscriptions = subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; @@ -707,6 +699,8 @@ async function watchSubscriptions() { let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount); let current_delay = 0; + + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); for (let i = 0; i < valid_subscriptions.length; i++) { let sub = valid_subscriptions[i]; @@ -2026,7 +2020,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { let files = null; let playlists = null; - let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getAllSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; + let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; // get basic info depending on multi-user mode being enabled if (req.isAuthenticated()) { @@ -2456,11 +2450,11 @@ app.post('/api/updateSubscription', optionalJwt, async (req, res) => { }); }); -app.post('/api/getAllSubscriptions', optionalJwt, async (req, res) => { +app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { let user_uid = req.isAuthenticated() ? req.user.uid : null; // get subs from api - let subscriptions = subscriptions_api.getAllSubscriptions(user_uid); + let subscriptions = subscriptions_api.getSubscriptions(user_uid); res.send({ subscriptions: subscriptions diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 396516d..dfcff58 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -255,10 +255,6 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, } async function getVideosForSub(sub, user_uid = null) { - if (!subExists(sub.id, user_uid)) { - return false; - } - // get sub_db let sub_db = null; if (user_uid) @@ -266,6 +262,13 @@ async function getVideosForSub(sub, user_uid = null) { else sub_db = db.get('subscriptions').find({id: sub.id}); + const latest_sub_obj = sub_db.value(); + if (!latest_sub_obj || latest_sub_obj['downloading']) { + return false; + } + + updateSubscriptionProperty(sub, {downloading: true}, user_uid); + // get basePath let basePath = null; if (user_uid) @@ -290,6 +293,7 @@ async function getVideosForSub(sub, user_uid = null) { return new Promise(resolve => { youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) { + updateSubscriptionProperty(sub, {downloading: false}, user_uid); logger.verbose('Subscription: finished check for ' + sub.name); if (err && !output) { logger.error(err.stderr ? err.stderr : err.message); @@ -319,6 +323,7 @@ async function getVideosForSub(sub, user_uid = null) { if (output.length === 0 || (output.length === 1 && output[0] === '')) { logger.verbose('No additional videos to download for ' + sub.name); resolve(true); + return; } for (let i = 0; i < output.length; i++) { let output_json = null; @@ -345,6 +350,7 @@ async function getVideosForSub(sub, user_uid = null) { }); }, err => { logger.error(err); + updateSubscriptionProperty(sub, {downloading: false}, user_uid); }); } @@ -460,13 +466,28 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ } } -function getAllSubscriptions(user_uid = null) { +function getSubscriptions(user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').value(); else return db.get('subscriptions').value(); } +function getAllSubscriptions() { + let subscriptions = null; + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + if (multiUserMode) { + subscriptions = []; + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); + } + } else { + subscriptions = subscriptions_api.getSubscriptions(); + } + return subscriptions; +} + function getSubscription(subID, user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); @@ -490,6 +511,21 @@ function updateSubscription(sub, user_uid = null) { return true; } +function updateSubscriptionPropertyMultiple(subs, assignment_obj) { + subs.forEach(sub => { + updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); + }); +} + +function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { + if (user_uid) { + users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); + } else { + db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); + } + return true; +} + function subExists(subID, user_uid = null) { if (user_uid) return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); @@ -580,6 +616,7 @@ async function removeIDFromArchive(archive_path, id) { module.exports = { getSubscription : getSubscription, getSubscriptionByName : getSubscriptionByName, + getSubscriptions : getSubscriptions, getAllSubscriptions : getAllSubscriptions, updateSubscription : updateSubscription, subscribe : subscribe, @@ -588,5 +625,6 @@ module.exports = { getVideosForSub : getVideosForSub, removeIDFromArchive : removeIDFromArchive, setLogger : setLogger, - initialize : initialize + initialize : initialize, + updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple } diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 77acefc..4ae90dd 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -374,7 +374,7 @@ export class PostsService implements CanActivate { } getAllSubscriptions() { - return this.http.post(this.path + 'getAllSubscriptions', {}, this.httpOptions); + return this.http.post(this.path + 'getSubscriptions', {}, this.httpOptions); } // current downloads diff --git a/src/app/subscription/subscription/subscription.component.html b/src/app/subscription/subscription/subscription.component.html index e404df3..82747ac 100644 --- a/src/app/subscription/subscription/subscription.component.html +++ b/src/app/subscription/subscription/subscription.component.html @@ -4,6 +4,7 @@

{{subscription.name}} (Paused)

+

diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index df0561d..5f32f40 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnDestroy, OnInit } from '@angular/core'; import { PostsService } from 'app/posts.services'; import { ActivatedRoute, Router, ParamMap } from '@angular/router'; import { MatDialog } from '@angular/material/dialog'; @@ -9,7 +9,7 @@ import { EditSubscriptionDialogComponent } from 'app/dialogs/edit-subscription-d templateUrl: './subscription.component.html', styleUrls: ['./subscription.component.scss'] }) -export class SubscriptionComponent implements OnInit { +export class SubscriptionComponent implements OnInit, OnDestroy { id = null; subscription = null; @@ -44,22 +44,11 @@ export class SubscriptionComponent implements OnInit { }; filterProperty = this.filterProperties['upload_date']; downloading = false; - - initialized = false; + sub_interval = null; constructor(private postsService: PostsService, private route: ActivatedRoute, private router: Router, private dialog: MatDialog) { } ngOnInit() { - this.route.paramMap.subscribe((params: ParamMap) => { - this.id = params.get('id'); - this.postsService.service_initialized.subscribe(init => { - if (init) { - this.initialized = true; - this.getConfig(); - this.getSubscription(); - } - }); - }); if (this.route.snapshot.paramMap.get('id')) { this.id = this.route.snapshot.paramMap.get('id'); @@ -67,6 +56,7 @@ export class SubscriptionComponent implements OnInit { if (init) { this.getConfig(); this.getSubscription(); + this.sub_interval = setInterval(() => this.getSubscription(), 1000); } }); } @@ -78,6 +68,13 @@ export class SubscriptionComponent implements OnInit { } } + ngOnDestroy() { + // prevents subscription getter from running in the background + if (this.sub_interval) { + clearInterval(this.sub_interval); + } + } + goBack() { this.router.navigate(['/subscriptions']); } From 3f1532b4c609951b1998ef8fca7b7f6bea9bac0c Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 22 Dec 2020 01:23:43 -0500 Subject: [PATCH 14/23] Updated migration - Fixed bug in migration process for single-user mode - Changed name of migration Removed unused code for getmp3/mp4 and fixed bug when retrieving playlist if it didn't exist Fixed bug in streaming code where playlist audio files would not play if the file path was not present Fixed bug in getallsubscriptions for single user mode --- backend/app.js | 32 ++++++++++---------------------- backend/subscriptions.js | 2 +- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/backend/app.js b/backend/app.js index e1790fd..98a054e 100644 --- a/backend/app.js +++ b/backend/app.js @@ -212,8 +212,8 @@ async function checkMigrations() { // 4.1->4.2 migration - const add_description_migration_complete = db.get('add_description_migration_complete').value(); - if (!add_description_migration_complete) { + const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value(); + if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); success = success && await addMetadataPropertyToDB('view_count'); @@ -276,12 +276,12 @@ async function simplifyDBFileStructure() { } if (db.get('files.video').value() !== undefined && db.get('files.audio').value() !== undefined) { - const files = db.get('files.video').value().concat(db.get('files.audio')); + const files = db.get('files.video').value().concat(db.get('files.audio').value()); db.assign({files: files}).write(); } if (db.get('playlists.video').value() !== undefined && db.get('playlists.audio').value() !== undefined) { - const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio')); + const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio').value()); db.assign({playlists: playlists}).write(); } @@ -303,7 +303,7 @@ async function addMetadataPropertyToDB(property_key) { } // sets migration to complete - db.set('add_description_migration_complete', true).write(); + db.set('simplified_db_migration_complete', true).write(); return true; } catch(err) { logger.error(err); @@ -1935,8 +1935,8 @@ async function addThumbnails(files) { // gets all download mp3s app.get('/api/getMp3s', optionalJwt, async function(req, res) { - var mp3s = db.get('files').chain().find({isAudio: true}).value(); // getMp3s(); - var playlists = db.get('playlists.audio').value(); + var mp3s = db.get('files').value().filter(file => file.isAudio === true); + var playlists = db.get('playlists').value(); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user audio files/playlists @@ -1947,12 +1947,6 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { mp3s = JSON.parse(JSON.stringify(mp3s)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(mp3s); - } - - res.send({ mp3s: mp3s, playlists: playlists @@ -1961,7 +1955,7 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { // gets all download mp4s app.get('/api/getMp4s', optionalJwt, async function(req, res) { - var mp4s = db.get('files').chain().find({isAudio: false}).value(); // getMp4s(); + var mp4s = db.get('files').value().filter(file => file.isAudio === false); var playlists = db.get('playlists').value(); const is_authenticated = req.isAuthenticated(); @@ -1974,11 +1968,6 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) { mp4s = JSON.parse(JSON.stringify(mp4s)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(mp4s); - } - res.send({ mp4s: mp4s, playlists: playlists @@ -2501,14 +2490,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { if (req.isAuthenticated()) { playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID); - type = playlist.type; } else { playlist = db.get(`playlists`).find({id: playlistID}).value(); } res.send({ playlist: playlist, - type: type, + type: playlist && playlist.type, success: !!playlist }); }); @@ -2746,7 +2734,7 @@ app.get('/api/stream/:id', optionalJwt, (req, res) => { } if (!file_path) { - file_path = path.join(videoFolderPath, id + ext); + file_path = path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); } const stat = fs.statSync(file_path) diff --git a/backend/subscriptions.js b/backend/subscriptions.js index dfcff58..4bdca93 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -483,7 +483,7 @@ function getAllSubscriptions() { if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); } } else { - subscriptions = subscriptions_api.getSubscriptions(); + subscriptions = getSubscriptions(); } return subscriptions; } From 88a1c310901ad6ca6e3dcb8241ebbffc2b9eff96 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 22 Dec 2020 01:24:27 -0500 Subject: [PATCH 15/23] Removed unused code in home page --- src/app/main/main.component.html | 89 ------------------- src/app/main/main.component.ts | 144 ------------------------------- 2 files changed, 233 deletions(-) diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index 59956c8..9cb905e 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -187,92 +187,3 @@

Custom playlists

- - \ No newline at end of file diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 013d0f3..30694bf 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -8,7 +8,6 @@ import { MatSnackBar } from '@angular/material/snack-bar'; import { saveAs } from 'file-saver'; import { YoutubeSearchService, Result } from '../youtube-search.service'; import { Router, ActivatedRoute } from '@angular/router'; -import { CreatePlaylistComponent } from 'app/create-playlist/create-playlist.component'; import { Platform } from '@angular/cdk/platform'; import { v4 as uuid } from 'uuid'; import { ArgModifierDialogComponent } from 'app/dialogs/arg-modifier-dialog/arg-modifier-dialog.component'; @@ -244,13 +243,6 @@ export class MainComponent implements OnInit { this.useDefaultDownloadingAgent = this.postsService.config['Advanced']['use_default_downloading_agent']; this.customDownloadingAgent = this.postsService.config['Advanced']['custom_downloading_agent']; - - - if (this.fileManagerEnabled) { - this.getMp3s(); - this.getMp4s(); - } - if (this.youtubeSearchEnabled && this.youtubeAPIKey) { this.youtubeSearch.initializeAPI(this.youtubeAPIKey); this.attachToInput(); @@ -335,61 +327,6 @@ export class MainComponent implements OnInit { this.setCols(); } - // file manager stuff - - getMp3s() { - this.postsService.getMp3s().subscribe(result => { - const mp3s = result['mp3s']; - const playlists = result['playlists']; - // if they are different - if (JSON.stringify(this.mp3s) !== JSON.stringify(mp3s)) { this.mp3s = mp3s }; - this.playlists.audio = playlists; - - // get thumbnail url by using first video. this is a temporary hack - for (let i = 0; i < this.playlists.audio.length; i++) { - const playlist = this.playlists.audio[i]; - let videoToExtractThumbnail = null; - for (let j = 0; j < this.mp3s.length; j++) { - if (this.mp3s[j].id === playlist.fileNames[0]) { - // found the corresponding file - videoToExtractThumbnail = this.mp3s[j]; - } - } - - if (videoToExtractThumbnail) { this.playlist_thumbnails[playlist.id] = videoToExtractThumbnail.thumbnailURL; } - } - }, error => { - console.log(error); - }); - } - - getMp4s() { - this.postsService.getMp4s().subscribe(result => { - const mp4s = result['mp4s']; - const playlists = result['playlists']; - // if they are different - if (JSON.stringify(this.mp4s) !== JSON.stringify(mp4s)) { this.mp4s = mp4s }; - this.playlists.video = playlists; - - // get thumbnail url by using first video. this is a temporary hack - for (let i = 0; i < this.playlists.video.length; i++) { - const playlist = this.playlists.video[i]; - let videoToExtractThumbnail = null; - for (let j = 0; j < this.mp4s.length; j++) { - if (this.mp4s[j].id === playlist.fileNames[0]) { - // found the corresponding file - videoToExtractThumbnail = this.mp4s[j]; - } - } - - if (videoToExtractThumbnail) { this.playlist_thumbnails[playlist.id] = videoToExtractThumbnail.thumbnailURL; } - } - }, - error => { - console.log(error); - }); - } - public setCols() { if (window.innerWidth <= 350) { this.files_cols = 1; @@ -437,44 +374,6 @@ export class MainComponent implements OnInit { return null; } - public removeFromMp3(name: string) { - for (let i = 0; i < this.mp3s.length; i++) { - if (this.mp3s[i].id === name || this.mp3s[i].id + '.mp3' === name) { - this.mp3s.splice(i, 1); - } - } - this.getMp3s(); - } - - public removePlaylistMp3(playlistID, index) { - this.postsService.removePlaylist(playlistID, 'audio').subscribe(res => { - if (res['success']) { - this.playlists.audio.splice(index, 1); - this.openSnackBar('Playlist successfully removed.', ''); - } - this.getMp3s(); - }); - } - - public removeFromMp4(name: string) { - for (let i = 0; i < this.mp4s.length; i++) { - if (this.mp4s[i].id === name || this.mp4s[i].id + '.mp4' === name) { - this.mp4s.splice(i, 1); - } - } - this.getMp4s(); - } - - public removePlaylistMp4(playlistID, index) { - this.postsService.removePlaylist(playlistID, 'video').subscribe(res => { - if (res['success']) { - this.playlists.video.splice(index, 1); - this.openSnackBar('Playlist successfully removed.', ''); - } - this.getMp4s(); - }); - } - // download helpers downloadHelperMp3(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { @@ -504,16 +403,6 @@ export class MainComponent implements OnInit { // remove download from current downloads this.removeDownloadFromCurrentDownloads(new_download); - - // reloads mp3s - if (this.fileManagerEnabled) { - this.getMp3s(); - setTimeout(() => { - this.audioFileCards.forEach(filecard => { - filecard.onHoverResponse(); - }); - }, 200); - } } downloadHelperMp4(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { @@ -543,16 +432,6 @@ export class MainComponent implements OnInit { // remove download from current downloads this.removeDownloadFromCurrentDownloads(new_download); - - // reloads mp4s - if (this.fileManagerEnabled) { - this.getMp4s(); - setTimeout(() => { - this.videoFileCards.forEach(filecard => { - filecard.onHoverResponse(); - }); - }, 200); - } } // download click handler @@ -745,8 +624,6 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded this.postsService.deleteFile(name, 'video').subscribe(delRes => { - // reload mp3s - this.getMp3s(); }); } }); @@ -762,8 +639,6 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded this.postsService.deleteFile(name, 'audio').subscribe(delRes => { - // reload mp4s - this.getMp4s(); }); } }); @@ -1110,25 +985,6 @@ export class MainComponent implements OnInit { } } - // creating a playlist - openCreatePlaylistDialog(type) { - const dialogRef = this.dialog.open(CreatePlaylistComponent, { - data: { - filesToSelectFrom: (type === 'audio') ? this.mp3s : this.mp4s, - type: type - } - }); - dialogRef.afterClosed().subscribe(result => { - if (result) { - if (type === 'audio') { this.getMp3s() }; - if (type === 'video') { this.getMp4s() }; - this.openSnackBar('Successfully created playlist!', ''); - } else if (result === false) { - this.openSnackBar('ERROR: failed to create playlist!', ''); - } - }); - } - // modify custom args openArgsModifierDialog() { const dialogRef = this.dialog.open(ArgModifierDialogComponent, { From 2656147570910613881add021cbaf990ec6f3db7 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 22 Dec 2020 01:24:50 -0500 Subject: [PATCH 16/23] Optimized get/set subscription process --- .../edit-subscription-dialog.component.ts | 6 +++++- .../subscription/subscription.component.ts | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts index b1d2dd4..19ff03b 100644 --- a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts +++ b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts @@ -61,9 +61,13 @@ export class EditSubscriptionDialogComponent implements OnInit { ]; constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialog: MatDialog, private postsService: PostsService) { - this.sub = this.data.sub; + this.sub = JSON.parse(JSON.stringify(this.data.sub)); this.new_sub = JSON.parse(JSON.stringify(this.sub)); + // ignore videos to keep requests small + delete this.sub['videos']; + delete this.new_sub['videos']; + this.audioOnlyMode = this.sub.type === 'audio'; this.download_all = !this.sub.timerange; diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index 5f32f40..461dcfc 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -56,7 +56,7 @@ export class SubscriptionComponent implements OnInit, OnDestroy { if (init) { this.getConfig(); this.getSubscription(); - this.sub_interval = setInterval(() => this.getSubscription(), 1000); + this.sub_interval = setInterval(() => this.getSubscription(true), 1000); } }); } @@ -79,8 +79,14 @@ export class SubscriptionComponent implements OnInit, OnDestroy { this.router.navigate(['/subscriptions']); } - getSubscription() { + getSubscription(low_cost = false) { this.postsService.getSubscription(this.id).subscribe(res => { + if (low_cost && res['subscription'].videos.length === this.subscription?.videos.length) { + if (res['subscription']['downloading'] !== this.subscription['downloading']) { + this.subscription['downloading'] = res['subscription']['downloading']; + } + return; + } this.subscription = res['subscription']; this.files = res['files']; if (this.search_mode) { From 6eb6ffa5e409b817bae494c9486c6cc15877c301 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 22 Dec 2020 01:25:12 -0500 Subject: [PATCH 17/23] Get user videos now accepts an optional type parameter --- backend/authentication/auth.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 3c77edd..ce8f100 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -281,9 +281,9 @@ exports.adminExists = function() { // video stuff -exports.getUserVideos = function(user_uid) { +exports.getUserVideos = function(user_uid, type) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['files']; + return type ? user['files'].filter(file => file.isAudio = (type === 'audio')) : user['files']; } exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { From 9a57080bb32dd7553caeb437a7cd581372541df9 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Wed, 23 Dec 2020 01:24:43 -0500 Subject: [PATCH 18/23] Category is now properly stored in the database --- backend/db.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/db.js b/backend/db.js index b7298d0..38dfdcb 100644 --- a/backend/db.js +++ b/backend/db.js @@ -15,7 +15,7 @@ function initialize(input_db, input_users_db, input_logger) { setLogger(input_logger); } -function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null) { +function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null, category = null) { let db_path = null; const file_id = file_path.substring(0, file_path.length-4); const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); @@ -29,6 +29,9 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null, custo // add thumbnail path file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path); + // if category exists, only include essential info + if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; + if (!sub) { if (multiUserMode) { const user_uid = multiUserMode.user; From c63a64ebefacdf6f0514ae0d4756964cf896fdc7 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Wed, 23 Dec 2020 01:29:22 -0500 Subject: [PATCH 19/23] Categories will now auto-generate playlists --- backend/app.js | 35 +++++++++++++-- backend/authentication/auth.js | 44 ++++++++++++++++--- .../custom-playlists.component.ts | 2 +- .../recent-videos/recent-videos.component.ts | 2 +- .../unified-file-card.component.html | 9 +++- .../unified-file-card.component.scss | 5 +++ .../video-info-dialog.component.html | 2 +- src/app/player/player.component.ts | 4 ++ 8 files changed, 88 insertions(+), 15 deletions(-) diff --git a/backend/app.js b/backend/app.js index 98a054e..d284b3b 100644 --- a/backend/app.js +++ b/backend/app.js @@ -79,7 +79,7 @@ const logger = winston.createLogger({ }); config_api.initialize(logger); -auth_api.initialize(users_db, logger); +auth_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger); subscriptions_api.initialize(db, users_db, logger, db_api); categories_api.initialize(db, users_db, logger, db_api); @@ -1215,7 +1215,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; // registers file in DB - file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath); + file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category); if (file_name) file_names.push(file_name); } @@ -2014,10 +2014,37 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { // get basic info depending on multi-user mode being enabled if (req.isAuthenticated()) { files = auth_api.getUserVideos(req.user.uid); - playlists = auth_api.getUserPlaylists(req.user.uid); + playlists = auth_api.getUserPlaylists(req.user.uid, files); } else { files = db.get('files').value(); - playlists = db.get('playlists').value(); + playlists = JSON.parse(JSON.stringify(db.get('playlists').value())); + const categories = db.get('categories').value(); + if (categories) { + categories.forEach(category => { + const audio_files = files && files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); + const video_files = files && files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); + if (audio_files && audio_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: audio_files[0].thumbnailURL, + thumbnailPath: audio_files[0].thumbnailPath, + fileNames: audio_files.map(file => file.id), + type: 'audio', + auto: true + }); + } + if (video_files && video_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: video_files[0].thumbnailURL, + thumbnailPath: video_files[0].thumbnailPath, + fileNames: video_files.map(file => file.id), + type: 'video', + auto: true + }); + } + }); + } } // loop through subscriptions and add videos diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index ce8f100..e5f04ec 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -15,15 +15,16 @@ var JwtStrategy = require('passport-jwt').Strategy, // other required vars let logger = null; -var users_db = null; +let db = null; +let users_db = null; let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function(input_users_db, input_logger) { +exports.initialize = function(input_db, input_users_db, input_logger) { setLogger(input_logger) - setDB(input_users_db); + setDB(input_db, input_users_db); /************************* * Authentication module @@ -61,7 +62,8 @@ function setLogger(input_logger) { logger = input_logger; } -function setDB(input_users_db) { +function setDB(input_db, input_users_db) { + db = input_db; users_db = input_users_db; } @@ -310,9 +312,39 @@ exports.removePlaylist = function(user_uid, playlistID) { return true; } -exports.getUserPlaylists = function(user_uid) { +exports.getUserPlaylists = function(user_uid, user_files = null) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['playlists']; + const playlists = JSON.parse(JSON.stringify(user['playlists'])); + const categories = db.get('categories').value(); + if (categories && user_files) { + categories.forEach(category => { + const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); + const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); + if (audio_files && audio_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: audio_files[0].thumbnailURL, + thumbnailPath: audio_files[0].thumbnailPath, + fileNames: audio_files.map(file => file.id), + type: 'audio', + uid: user_uid, + auto: true + }); + } + if (video_files && video_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: video_files[0].thumbnailURL, + thumbnailPath: video_files[0].thumbnailPath, + fileNames: video_files.map(file => file.id), + type: 'video', + uid: user_uid, + auto: true + }); + } + }); + } + return playlists; } exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index d2d65d6..73e3036 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -62,7 +62,7 @@ export class CustomPlaylistsComponent implements OnInit { } else { localStorage.setItem('player_navigator', this.router.url); const fileNames = playlist.fileNames; - this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID}]); + this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID, auto: playlist.auto}]); } } else { // playlist not found diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index 7ca6c9b..83bd72f 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -98,7 +98,7 @@ export class RecentVideosComponent implements OnInit { private filterFiles(value: string) { const filterValue = value.toLowerCase(); - this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.toLowerCase().includes(filterValue)); + this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.name?.toLowerCase().includes(filterValue)); this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}); } diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index a080324..066d78e 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -1,5 +1,10 @@
-
{{(file_obj.type === 'audio' || file_obj.isAudio) ? 'audiotrack' : 'movie'}}  {{file_obj.registered | date:'shortDate' : undefined : locale.ngID}}
+
+ {{(file_obj.type === 'audio' || file_obj.isAudio) ? 'audiotrack' : 'movie'}} +    + Auto-generated + {{file_obj.registered | date:'shortDate' : undefined : locale.ngID}} +
- + diff --git a/src/app/components/unified-file-card/unified-file-card.component.scss b/src/app/components/unified-file-card/unified-file-card.component.scss index 067ef4a..8d8ea1a 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.scss +++ b/src/app/components/unified-file-card/unified-file-card.component.scss @@ -111,6 +111,11 @@ top: 1px; left: 5px; z-index: 99999; + width: calc(100% - 8px); + white-space: nowrap; + overflow: hidden; + display: block; + text-overflow: ellipsis; } .audio-video-icon { diff --git a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html index 203cfd5..e62d1c9 100644 --- a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html +++ b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html @@ -27,7 +27,7 @@
Category: 
-
{{file.category}}N/A
+
{{file.category.name}}N/A
diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index c2aa1c2..123e2b4 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -207,6 +207,10 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { } getPlaylistFiles() { + if (this.route.snapshot.paramMap.get('auto') === 'true') { + this.show_player = true; + return; + } this.postsService.getPlaylist(this.id, null, this.uuid).subscribe(res => { if (res['playlist']) { this.db_playlist = res['playlist']; From 48350936067417729de11054d55304097861b68e Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 24 Dec 2020 00:10:54 -0500 Subject: [PATCH 20/23] Fixed issue where some non-YT videos would fail as the pre-check was incompatible --- backend/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app.js b/backend/app.js index d284b3b..8c032c4 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1119,10 +1119,10 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // get video info prior to download let info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info) { + if (!info && url.includes('youtu')) { resolve(false); return; - } else { + } else if (info) { // check if it fits into a category. If so, then get info again using new downloadConfig category = await categories_api.categorize(info); From c08993e20bd471135a49bd53a811a7ff95fe5b74 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 24 Dec 2020 02:02:05 -0500 Subject: [PATCH 21/23] Old database files are now backed up prior to migration to simplified structure --- backend/app.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/app.js b/backend/app.js index 8c032c4..8730f36 100644 --- a/backend/app.js +++ b/backend/app.js @@ -260,6 +260,13 @@ async function runFilesToDBMigration() { } async function simplifyDBFileStructure() { + // back up db files + const old_db_file = fs.readJSONSync('./appdata/db.json'); + const old_users_db_file = fs.readJSONSync('./appdata/users.json'); + fs.writeJSONSync('appdata/db.old.json', old_db_file); + fs.writeJSONSync('appdata/users.old.json', old_users_db_file); + + // simplify let users = users_db.get('users').value(); for (let i = 0; i < users.length; i++) { const user = users[i]; From 33fc74b7e763178fa3eee60ef6995f9641b0b9aa Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 24 Dec 2020 02:04:43 -0500 Subject: [PATCH 22/23] Updated dev config --- src/assets/default.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/assets/default.json b/src/assets/default.json index 893896f..532b32e 100644 --- a/src/assets/default.json +++ b/src/assets/default.json @@ -26,7 +26,9 @@ "use_API_key": false, "API_key": "", "use_youtube_API": false, - "youtube_API_key": "" + "youtube_API_key": "", + "use_twitch_API": false, + "twitch_API_key": "" }, "Themes": { "default_theme": "default", @@ -57,7 +59,8 @@ "allow_advanced_download": true, "jwt_expiration": 86400, "logger_level": "debug", - "use_cookies": false + "use_cookies": false, + "default_downloader": "youtube-dlc" } } } \ No newline at end of file From 0e7bc1979fe4e4ed8139c3bb5d386a102243f0fd Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 24 Dec 2020 02:05:30 -0500 Subject: [PATCH 23/23] Updated versioning info --- backend/consts.js | 2 +- package.json | 2 +- src/app/consts.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/consts.js b/backend/consts.js index 64e6e09..fa14171 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -196,5 +196,5 @@ AVAILABLE_PERMISSIONS = [ module.exports = { CONFIG_ITEMS: CONFIG_ITEMS, AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS, - CURRENT_VERSION: 'v4.1' + CURRENT_VERSION: 'v4.2' } diff --git a/package.json b/package.json index e34f8a8..ab15811 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "youtube-dl-material", - "version": "4.1.0", + "version": "4.2.0", "license": "MIT", "scripts": { "ng": "ng", diff --git a/src/app/consts.ts b/src/app/consts.ts index ad0197d..950aff6 100644 --- a/src/app/consts.ts +++ b/src/app/consts.ts @@ -1 +1 @@ -export const CURRENT_VERSION = 'v4.1'; +export const CURRENT_VERSION = 'v4.2';