diff --git a/Public API v1.yaml b/Public API v1.yaml index e82328a..b57c386 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -261,12 +261,12 @@ paths: $ref: '#/components/schemas/inline_response_200_10' security: - Auth query parameter: [] - /api/getAllSubscriptions: + /api/getSubscriptions: post: tags: - subscriptions summary: Get all subscriptions - operationId: post-api-getAllSubscriptions + operationId: post-api-getSubscriptions requestBody: content: application/json: diff --git a/backend/app.js b/backend/app.js index d2c4378..8730f36 100644 --- a/backend/app.js +++ b/backend/app.js @@ -13,7 +13,7 @@ var express = require("express"); var bodyParser = require("body-parser"); var archiver = require('archiver'); var unzipper = require('unzipper'); -var db_api = require('./db') +var db_api = require('./db'); var utils = require('./utils') var mergeFiles = require('merge-files'); const low = require('lowdb') @@ -79,7 +79,7 @@ const logger = winston.createLogger({ }); config_api.initialize(logger); -auth_api.initialize(users_db, logger); +auth_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger); subscriptions_api.initialize(db, users_db, logger, db_api); categories_api.initialize(db, users_db, logger, db_api); @@ -87,14 +87,8 @@ categories_api.initialize(db, users_db, logger, db_api); // Set some defaults db.defaults( { - playlists: { - audio: [], - video: [] - }, - files: { - audio: [], - video: [] - }, + playlists: [], + files: [], configWriteFlag: false, downloads: {}, subscriptions: [], @@ -216,6 +210,20 @@ async function checkMigrations() { else { logger.error('Migration failed: 3.5->3.6+'); } } + // 4.1->4.2 migration + + const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value(); + if (!simplified_db_migration_complete) { + logger.info('Beginning migration: 4.1->4.2+') + let success = await simplifyDBFileStructure(); + success = success && await addMetadataPropertyToDB('view_count'); + success = success && await addMetadataPropertyToDB('description'); + success = success && await addMetadataPropertyToDB('height'); + success = success && await addMetadataPropertyToDB('abr'); + if (success) { logger.info('4.1->4.2+ migration complete!'); } + else { logger.error('Migration failed: 4.1->4.2+'); } + } + return true; } @@ -251,6 +259,65 @@ async function runFilesToDBMigration() { } } +async function simplifyDBFileStructure() { + // back up db files + const old_db_file = fs.readJSONSync('./appdata/db.json'); + const old_users_db_file = fs.readJSONSync('./appdata/users.json'); + fs.writeJSONSync('appdata/db.old.json', old_db_file); + fs.writeJSONSync('appdata/users.old.json', old_users_db_file); + + // simplify + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + const user = users[i]; + if (user['files']['video'] !== undefined && user['files']['audio'] !== undefined) { + const user_files = user['files']['video'].concat(user['files']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({files: user_files}).write(); + } + if (user['playlists']['video'] !== undefined && user['playlists']['audio'] !== undefined) { + const user_playlists = user['playlists']['video'].concat(user['playlists']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({playlists: user_playlists}).write(); + } + } + + if (db.get('files.video').value() !== undefined && db.get('files.audio').value() !== undefined) { + const files = db.get('files.video').value().concat(db.get('files.audio').value()); + db.assign({files: files}).write(); + } + + if (db.get('playlists.video').value() !== undefined && db.get('playlists.audio').value() !== undefined) { + const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio').value()); + db.assign({playlists: playlists}).write(); + } + + + return true; +} + +async function addMetadataPropertyToDB(property_key) { + try { + const dirs_to_check = db_api.getFileDirectoriesAndDBs(); + for (const dir_to_check of dirs_to_check) { + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); + for (const file of files) { + if (file[property_key]) { + dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); + } + } + } + + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); + return true; + } catch(err) { + logger.error(err); + return false; + } +} + async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku @@ -560,11 +627,17 @@ async function loadConfig() { // creates archive path if missing await fs.ensureDir(archivePath); + // check migrations + await checkMigrations(); + // now this is done here due to youtube-dl's repo takedown await startYoutubeDL(); // get subscriptions if (allowSubscriptions) { + // set downloading to false + let subscriptions = subscriptions_api.getAllSubscriptions(); + subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds watchSubscriptions(); setInterval(() => { @@ -574,9 +647,6 @@ async function loadConfig() { db_api.importUnregisteredFiles(); - // check migrations - await checkMigrations(); - // load in previous downloads downloads = db.get('downloads').value(); @@ -626,27 +696,20 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) { } async function watchSubscriptions() { - let subscriptions = null; - - const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode) { - subscriptions = []; - let users = users_db.get('users').value(); - for (let i = 0; i < users.length; i++) { - if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); - } - } else { - subscriptions = subscriptions_api.getAllSubscriptions(); - } + let subscriptions = subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; - let subscriptions_amount = subscriptions.length; + const valid_subscriptions = subscriptions.filter(sub => !sub.paused); + + let subscriptions_amount = valid_subscriptions.length; let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount); let current_delay = 0; - for (let i = 0; i < subscriptions.length; i++) { - let sub = subscriptions[i]; + + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + for (let i = 0; i < valid_subscriptions.length; i++) { + let sub = valid_subscriptions[i]; // don't check the sub if the last check for the same subscription has not completed if (subscription_timeouts[sub.id]) { @@ -700,64 +763,6 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -async function getMp3s() { - let mp3s = []; - var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(audioFolderPath.length, file.length); - - var stats = await fs.stat(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp3(id, audioFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = true; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp3s.push(file_obj); - } - return mp3s; -} - -async function getMp4s(relative_path = true) { - let mp4s = []; - var files = await utils.recFindByExt(videoFolderPath, 'mp4'); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(videoFolderPath.length, file.length); - - var stats = fs.statSync(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp4(id, videoFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - - var size = stats.size; - - var isaudio = false; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp4s.push(file_obj); - } - return mp4s; -} - function getThumbnailMp3(name) { var obj = utils.getJSONMp3(name, audioFolderPath); @@ -1121,10 +1126,10 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // get video info prior to download let info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info) { + if (!info && url.includes('youtu')) { resolve(false); return; - } else { + } else if (info) { // check if it fits into a category. If so, then get info again using new downloadConfig category = await categories_api.categorize(info); @@ -1217,7 +1222,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; // registers file in DB - file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath); + file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category); if (file_name) file_names.push(file_name); } @@ -1937,8 +1942,8 @@ async function addThumbnails(files) { // gets all download mp3s app.get('/api/getMp3s', optionalJwt, async function(req, res) { - var mp3s = db.get('files.audio').value(); // getMp3s(); - var playlists = db.get('playlists.audio').value(); + var mp3s = db.get('files').value().filter(file => file.isAudio === true); + var playlists = db.get('playlists').value(); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { // get user audio files/playlists @@ -1949,12 +1954,6 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { mp3s = JSON.parse(JSON.stringify(mp3s)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(mp3s); - } - - res.send({ mp3s: mp3s, playlists: playlists @@ -1963,8 +1962,8 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) { // gets all download mp4s app.get('/api/getMp4s', optionalJwt, async function(req, res) { - var mp4s = db.get('files.video').value(); // getMp4s(); - var playlists = db.get('playlists.video').value(); + var mp4s = db.get('files').value().filter(file => file.isAudio === false); + var playlists = db.get('playlists').value(); const is_authenticated = req.isAuthenticated(); if (is_authenticated) { @@ -1976,11 +1975,6 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) { mp4s = JSON.parse(JSON.stringify(mp4s)); - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - // add thumbnails if present - // await addThumbnails(mp4s); - } - res.send({ mp4s: mp4s, playlists: playlists @@ -1995,21 +1989,11 @@ app.post('/api/getFile', optionalJwt, function (req, res) { var file = null; if (req.isAuthenticated()) { - file = auth_api.getUserVideo(req.user.uid, uid, type); + file = auth_api.getUserVideo(req.user.uid, uid); } else if (uuid) { - file = auth_api.getUserVideo(uuid, uid, type, true); + file = auth_api.getUserVideo(uuid, uid, true); } else { - if (!type) { - file = db.get('files.audio').find({uid: uid}).value(); - if (!file) { - file = db.get('files.video').find({uid: uid}).value(); - if (file) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!file && type) file = db.get(`files.${type}`).find({uid: uid}).value(); + file = db.get('files').find({uid: uid}).value(); } // check if chat exists for twitch videos @@ -2029,32 +2013,47 @@ app.post('/api/getFile', optionalJwt, function (req, res) { app.post('/api/getAllFiles', optionalJwt, async function (req, res) { // these are returned - let files = []; - let playlists = []; - let subscription_files = []; + let files = null; + let playlists = null; - let videos = null; - let audios = null; - let audio_playlists = null; - let video_playlists = null; - let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getAllSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; + let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; // get basic info depending on multi-user mode being enabled if (req.isAuthenticated()) { - videos = auth_api.getUserVideos(req.user.uid, 'video'); - audios = auth_api.getUserVideos(req.user.uid, 'audio'); - audio_playlists = auth_api.getUserPlaylists(req.user.uid, 'audio'); - video_playlists = auth_api.getUserPlaylists(req.user.uid, 'video'); + files = auth_api.getUserVideos(req.user.uid); + playlists = auth_api.getUserPlaylists(req.user.uid, files); } else { - videos = db.get('files.audio').value(); - audios = db.get('files.video').value(); - audio_playlists = db.get('playlists.audio').value(); - video_playlists = db.get('playlists.video').value(); + files = db.get('files').value(); + playlists = JSON.parse(JSON.stringify(db.get('playlists').value())); + const categories = db.get('categories').value(); + if (categories) { + categories.forEach(category => { + const audio_files = files && files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); + const video_files = files && files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); + if (audio_files && audio_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: audio_files[0].thumbnailURL, + thumbnailPath: audio_files[0].thumbnailPath, + fileNames: audio_files.map(file => file.id), + type: 'audio', + auto: true + }); + } + if (video_files && video_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: video_files[0].thumbnailURL, + thumbnailPath: video_files[0].thumbnailPath, + fileNames: video_files.map(file => file.id), + type: 'video', + auto: true + }); + } + }); + } } - files = videos.concat(audios); - playlists = video_playlists.concat(audio_playlists); - // loop through subscriptions and add videos for (let i = 0; i < subscriptions.length; i++) { sub = subscriptions[i]; @@ -2122,14 +2121,13 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { // video sharing app.post('/api/enableSharing', optionalJwt, function(req, res) { - var type = req.body.type; var uid = req.body.uid; var is_playlist = req.body.is_playlist; let success = false; // multi-user mode if (req.isAuthenticated()) { // if multi user mode, use this method instead - success = auth_api.changeSharingMode(req.user.uid, uid, type, is_playlist, true); + success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, true); res.send({success: success}); return; } @@ -2138,12 +2136,12 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files.${type}`) + db.get(`files`) .find({uid: uid}) .assign({sharingEnabled: true}) .write(); } else if (is_playlist) { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: uid}) .assign({sharingEnabled: true}) .write(); @@ -2172,7 +2170,7 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { // multi-user mode if (req.isAuthenticated()) { // if multi user mode, use this method instead - success = auth_api.changeSharingMode(req.user.uid, uid, type, is_playlist, false); + success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, false); res.send({success: success}); return; } @@ -2181,12 +2179,12 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { try { success = true; if (!is_playlist && type !== 'subscription') { - db.get(`files.${type}`) + db.get(`files`) .find({uid: uid}) .assign({sharingEnabled: false}) .write(); } else if (is_playlist) { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: uid}) .assign({sharingEnabled: false}) .write(); @@ -2207,6 +2205,27 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { }); }); +app.post('/api/incrementViewCount', optionalJwt, async (req, res) => { + let file_uid = req.body.file_uid; + let sub_id = req.body.sub_id; + let uuid = req.body.uuid; + + if (!uuid && req.isAuthenticated()) { + uuid = req.user.uid; + } + + const file_obj = await db_api.getVideo(file_uid, uuid, sub_id); + + const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0; + const new_view_count = current_view_count + 1; + + await db_api.setVideoProperty(file_uid, {local_view_count: new_view_count}, uuid, sub_id); + + res.send({ + success: true + }); +}); + // categories app.post('/api/getAllCategories', optionalJwt, async (req, res) => { @@ -2399,7 +2418,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { var size = stats.size; var isaudio = false; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr); parsed_files.push(file_obj); } } else { @@ -2421,7 +2440,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { if (subscription.videos) { for (let i = 0; i < subscription.videos.length; i++) { const video = subscription.videos[i]; - parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date)); + parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr)); } } res.send({ @@ -2454,11 +2473,11 @@ app.post('/api/updateSubscription', optionalJwt, async (req, res) => { }); }); -app.post('/api/getAllSubscriptions', optionalJwt, async (req, res) => { +app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { let user_uid = req.isAuthenticated() ? req.user.uid : null; // get subs from api - let subscriptions = subscriptions_api.getAllSubscriptions(user_uid); + let subscriptions = subscriptions_api.getSubscriptions(user_uid); res.send({ subscriptions: subscriptions @@ -2485,7 +2504,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { if (req.isAuthenticated()) { auth_api.addPlaylist(req.user.uid, new_playlist, type); } else { - db.get(`playlists.${type}`) + db.get(`playlists`) .push(new_playlist) .write(); } @@ -2499,31 +2518,19 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { app.post('/api/getPlaylist', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; - let type = req.body.type; let uuid = req.body.uuid; let playlist = null; if (req.isAuthenticated()) { - playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID, type); - type = playlist.type; + playlist = auth_api.getUserPlaylist(uuid ? uuid : req.user.uid, playlistID); } else { - if (!type) { - playlist = db.get('playlists.audio').find({id: playlistID}).value(); - if (!playlist) { - playlist = db.get('playlists.video').find({id: playlistID}).value(); - if (playlist) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!playlist) playlist = db.get(`playlists.${type}`).find({id: playlistID}).value(); + playlist = db.get(`playlists`).find({id: playlistID}).value(); } res.send({ playlist: playlist, - type: type, + type: playlist && playlist.type, success: !!playlist }); }); @@ -2531,14 +2538,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; let fileNames = req.body.fileNames; - let type = req.body.type; let success = false; try { if (req.isAuthenticated()) { - auth_api.updatePlaylistFiles(req.user.uid, playlistID, fileNames, type); + auth_api.updatePlaylistFiles(req.user.uid, playlistID, fileNames); } else { - db.get(`playlists.${type}`) + db.get(`playlists`) .find({id: playlistID}) .assign({fileNames: fileNames}) .write(); @@ -2564,15 +2570,14 @@ app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { let playlistID = req.body.playlistID; - let type = req.body.type; let success = null; try { if (req.isAuthenticated()) { - auth_api.removePlaylist(req.user.uid, playlistID, type); + auth_api.removePlaylist(req.user.uid, playlistID); } else { // removes playlist from playlists - db.get(`playlists.${type}`) + db.get(`playlists`) .remove({id: playlistID}) .write(); } @@ -2594,23 +2599,23 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { var blacklistMode = req.body.blacklistMode; if (req.isAuthenticated()) { - let success = auth_api.deleteUserFile(req.user.uid, uid, type, blacklistMode); + let success = await auth_api.deleteUserFile(req.user.uid, uid, blacklistMode); res.send(success); return; } - var file_obj = db.get(`files.${type}`).find({uid: uid}).value(); + var file_obj = db.get(`files`).find({uid: uid}).value(); var name = file_obj.id; var fullpath = file_obj ? file_obj.path : null; var wasDeleted = false; if (await fs.pathExists(fullpath)) { wasDeleted = type === 'audio' ? await deleteAudioFile(name, path.basename(fullpath), blacklistMode) : await deleteVideoFile(name, path.basename(fullpath), blacklistMode); - db.get('files.video').remove({uid: uid}).write(); - // wasDeleted = true; + db.get('files').remove({uid: uid}).write(); + wasDeleted = true; res.send(wasDeleted); } else if (video_obj) { - db.get('files.video').remove({uid: uid}).write(); + db.get('files').remove({uid: uid}).write(); wasDeleted = true; res.send(wasDeleted); } else { @@ -2746,7 +2751,7 @@ app.get('/api/stream/:id', optionalJwt, (req, res) => { var head; let optionalParams = url_api.parse(req.url,true).query; let id = decodeURIComponent(req.params.id); - let file_path = req.query.file_path ? decodeURIComponent(req.query.file_path) : null; + let file_path = req.query.file_path ? decodeURIComponent(req.query.file_path.split('?')[0]) : null; if (!file_path && (req.isAuthenticated() || req.can_watch)) { let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); if (optionalParams['subName']) { @@ -2763,7 +2768,7 @@ app.get('/api/stream/:id', optionalJwt, (req, res) => { } if (!file_path) { - file_path = path.join(videoFolderPath, id + ext); + file_path = path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); } const stat = fs.statSync(file_path) diff --git a/backend/appdata/default.json b/backend/appdata/default.json index 001f515..dca9eee 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -38,7 +38,8 @@ "Subscriptions": { "allow_subscriptions": true, "subscriptions_base_path": "subscriptions/", - "subscriptions_check_interval": "300" + "subscriptions_check_interval": "300", + "redownload_fresh_uploads": false }, "Users": { "base_path": "users/", diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 9fdc7e7..e5f04ec 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -15,15 +15,16 @@ var JwtStrategy = require('passport-jwt').Strategy, // other required vars let logger = null; -var users_db = null; +let db = null; +let users_db = null; let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function(input_users_db, input_logger) { +exports.initialize = function(input_db, input_users_db, input_logger) { setLogger(input_logger) - setDB(input_users_db); + setDB(input_db, input_users_db); /************************* * Authentication module @@ -61,7 +62,8 @@ function setLogger(input_logger) { logger = input_logger; } -function setDB(input_users_db) { +function setDB(input_db, input_users_db) { + db = input_db; users_db = input_users_db; } @@ -283,22 +285,11 @@ exports.adminExists = function() { exports.getUserVideos = function(user_uid, type) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['files'][type]; + return type ? user['files'].filter(file => file.isAudio = (type === 'audio')) : user['files']; } -exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false) { - let file = null; - if (!type) { - file = users_db.get('users').find({uid: user_uid}).get(`files.audio`).find({uid: file_uid}).value(); - if (!file) { - file = users_db.get('users').find({uid: user_uid}).get(`files.video`).find({uid: file_uid}).value(); - if (file) type = 'video'; - } else { - type = 'audio'; - } - } - - if (!file && type) file = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); +exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { + let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); // prevent unauthorized users from accessing the file info if (file && !file['sharingEnabled'] && requireSharing) file = null; @@ -306,38 +297,58 @@ exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false return file; } -exports.addPlaylist = function(user_uid, new_playlist, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).push(new_playlist).write(); +exports.addPlaylist = function(user_uid, new_playlist) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write(); return true; } -exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).assign({fileNames: new_filenames}); +exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames}); return true; } -exports.removePlaylist = function(user_uid, playlistID, type) { - users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).remove({id: playlistID}).write(); +exports.removePlaylist = function(user_uid, playlistID) { + users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write(); return true; } -exports.getUserPlaylists = function(user_uid, type) { +exports.getUserPlaylists = function(user_uid, user_files = null) { const user = users_db.get('users').find({uid: user_uid}).value(); - return user['playlists'][type]; + const playlists = JSON.parse(JSON.stringify(user['playlists'])); + const categories = db.get('categories').value(); + if (categories && user_files) { + categories.forEach(category => { + const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio); + const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio); + if (audio_files && audio_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: audio_files[0].thumbnailURL, + thumbnailPath: audio_files[0].thumbnailPath, + fileNames: audio_files.map(file => file.id), + type: 'audio', + uid: user_uid, + auto: true + }); + } + if (video_files && video_files.length > 0) { + playlists.push({ + name: category['name'], + thumbnailURL: video_files[0].thumbnailURL, + thumbnailPath: video_files[0].thumbnailPath, + fileNames: video_files.map(file => file.id), + type: 'video', + uid: user_uid, + auto: true + }); + } + }); + } + return playlists; } -exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing = false) { - let playlist = null; - if (!type) { - playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.audio`).find({id: playlistID}).value(); - if (!playlist) { - playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.video`).find({id: playlistID}).value(); - if (playlist) type = 'video'; - } else { - type = 'audio'; - } - } - if (!playlist) playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).value(); +exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { + let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value(); // prevent unauthorized users from accessing the file info if (requireSharing && !playlist['sharingEnabled']) playlist = null; @@ -345,21 +356,22 @@ exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing = return playlist; } -exports.registerUserFile = function(user_uid, file_object, type) { - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) +exports.registerUserFile = function(user_uid, file_object) { + users_db.get('users').find({uid: user_uid}).get(`files`) .remove({ path: file_object['path'] }).write(); - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) + users_db.get('users').find({uid: user_uid}).get(`files`) .push(file_object) .write(); } -exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) { +exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) { let success = false; - const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); + const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); if (file_obj) { + const type = file_obj.isAudio ? 'audio' : 'video'; const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const ext = type === 'audio' ? '.mp3' : '.mp4'; @@ -375,7 +387,7 @@ exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode } const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext); - users_db.get('users').find({uid: user_uid}).get(`files.${type}`) + users_db.get('users').find({uid: user_uid}).get(`files`) .remove({ uid: file_uid }).write(); @@ -424,11 +436,11 @@ exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode return success; } -exports.changeSharingMode = function(user_uid, file_uid, type, is_playlist, enabled) { +exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) { let success = false; const user_db_obj = users_db.get('users').find({uid: user_uid}); if (user_db_obj.value()) { - const file_db_obj = is_playlist ? user_db_obj.get(`playlists.${type}`).find({id: file_uid}) : user_db_obj.get(`files.${type}`).find({uid: file_uid}); + const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid}); if (file_db_obj.value()) { success = true; file_db_obj.assign({sharingEnabled: enabled}).write(); diff --git a/backend/config.js b/backend/config.js index b2aff63..4790e34 100644 --- a/backend/config.js +++ b/backend/config.js @@ -215,7 +215,8 @@ DEFAULT_CONFIG = { "Subscriptions": { "allow_subscriptions": true, "subscriptions_base_path": "subscriptions/", - "subscriptions_check_interval": "300" + "subscriptions_check_interval": "300", + "redownload_fresh_uploads": false }, "Users": { "base_path": "users/", diff --git a/backend/consts.js b/backend/consts.js index bffe486..fa14171 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -126,6 +126,10 @@ let CONFIG_ITEMS = { 'key': 'ytdl_subscriptions_check_interval', 'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval' }, + 'ytdl_subscriptions_redownload_fresh_uploads': { + 'key': 'ytdl_subscriptions_redownload_fresh_uploads', + 'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads' + }, // Users 'ytdl_users_base_path': { @@ -192,5 +196,5 @@ AVAILABLE_PERMISSIONS = [ module.exports = { CONFIG_ITEMS: CONFIG_ITEMS, AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS, - CURRENT_VERSION: 'v4.1' + CURRENT_VERSION: 'v4.2' } diff --git a/backend/db.js b/backend/db.js index a2ddb7a..38dfdcb 100644 --- a/backend/db.js +++ b/backend/db.js @@ -15,7 +15,7 @@ function initialize(input_db, input_users_db, input_logger) { setLogger(input_logger); } -function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null) { +function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null, category = null) { let db_path = null; const file_id = file_path.substring(0, file_path.length-4); const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); @@ -29,12 +29,15 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null, custo // add thumbnail path file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path); + // if category exists, only include essential info + if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; + if (!sub) { if (multiUserMode) { const user_uid = multiUserMode.user; - db_path = users_db.get('users').find({uid: user_uid}).get(`files.${type}`); + db_path = users_db.get('users').find({uid: user_uid}).get(`files`); } else { - db_path = db.get(`files.${type}`) + db_path = db.get(`files`); } } else { if (multiUserMode) { @@ -94,18 +97,18 @@ function generateFileObject(id, type, customPath = null, sub = null) { var thumbnail = jsonobj.thumbnail; var duration = jsonobj.duration; var isaudio = type === 'audio'; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); + var description = jsonobj.description; + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); return file_obj; } function updatePlaylist(playlist, user_uid) { let playlistID = playlist.id; - let type = playlist.type; let db_loc = null; if (user_uid) { - db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}); + db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}); } else { - db_loc = db.get(`playlists.${type}`).find({id: playlistID}); + db_loc = db.get(`playlists`).find({id: playlistID}); } db_loc.assign(playlist).write(); return true; @@ -115,7 +118,7 @@ function getAppendedBasePathSub(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } -async function importUnregisteredFiles() { +function getFileDirectoriesAndDBs() { let dirs_to_check = []; let subscriptions_to_check = []; const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode @@ -132,14 +135,14 @@ async function importUnregisteredFiles() { // add user's audio dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'audio'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files.audio'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'audio' }); // add user's video dir to check list dirs_to_check.push({ basePath: path.join(usersFileFolder, user.uid, 'video'), - dbPath: users_db.get('users').find({uid: user.uid}).get('files.video'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files'), type: 'video' }); } @@ -153,14 +156,14 @@ async function importUnregisteredFiles() { // add audio dir to check list dirs_to_check.push({ basePath: audioFolderPath, - dbPath: db.get('files.audio'), + dbPath: db.get('files'), type: 'audio' }); // add video dir to check list dirs_to_check.push({ basePath: videoFolderPath, - dbPath: db.get('files.video'), + dbPath: db.get('files'), type: 'video' }); } @@ -181,6 +184,12 @@ async function importUnregisteredFiles() { }); } + return dirs_to_check; +} + +async function importUnregisteredFiles() { + const dirs_to_check = getFileDirectoriesAndDBs(); + // run through check list and check each file to see if it's missing from the db for (const dir_to_check of dirs_to_check) { // recursively get all files in dir's path @@ -199,9 +208,28 @@ async function importUnregisteredFiles() { } +async function getVideo(file_uid, uuid, sub_id) { + const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; + const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); + return sub_db_path.find({uid: file_uid}).value(); +} + +async function setVideoProperty(file_uid, assignment_obj, uuid, sub_id) { + const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; + const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); + const file_db_path = sub_db_path.find({uid: file_uid}); + if (!(file_db_path.value())) { + logger.error(`Failed to find file with uid ${file_uid}`); + } + sub_db_path.find({uid: file_uid}).assign(assignment_obj).write(); +} + module.exports = { initialize: initialize, registerFileDB: registerFileDB, updatePlaylist: updatePlaylist, - importUnregisteredFiles: importUnregisteredFiles + getFileDirectoriesAndDBs: getFileDirectoriesAndDBs, + importUnregisteredFiles: importUnregisteredFiles, + getVideo: getVideo, + setVideoProperty: setVideoProperty } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 9ab7542..4bdca93 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -255,10 +255,6 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, } async function getVideosForSub(sub, user_uid = null) { - if (!subExists(sub.id, user_uid)) { - return false; - } - // get sub_db let sub_db = null; if (user_uid) @@ -266,6 +262,13 @@ async function getVideosForSub(sub, user_uid = null) { else sub_db = db.get('subscriptions').find({id: sub.id}); + const latest_sub_obj = sub_db.value(); + if (!latest_sub_obj || latest_sub_obj['downloading']) { + return false; + } + + updateSubscriptionProperty(sub, {downloading: true}, user_uid); + // get basePath let basePath = null; if (user_uid) @@ -273,10 +276,7 @@ async function getVideosForSub(sub, user_uid = null) { else basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - - let appendedBasePath = null - appendedBasePath = getAppendedBasePath(sub, basePath); + let appendedBasePath = getAppendedBasePath(sub, basePath); let multiUserMode = null; if (user_uid) { @@ -286,14 +286,94 @@ async function getVideosForSub(sub, user_uid = null) { } } - const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' + const downloadConfig = await generateArgsForSubscription(sub, user_uid); + + // get videos + logger.verbose('Subscription: getting videos for subscription ' + sub.name); + + return new Promise(resolve => { + youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) { + updateSubscriptionProperty(sub, {downloading: false}, user_uid); + logger.verbose('Subscription: finished check for ' + sub.name); + if (err && !output) { + logger.error(err.stderr ? err.stderr : err.message); + if (err.stderr.includes('This video is unavailable')) { + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + const outputs = err.stdout.split(/\r\n|\r|\n/); + for (let i = 0; i < outputs.length; i++) { + const output = JSON.parse(outputs[i]); + handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) + if (err.stderr.includes(output['id']) && archive_path) { + // we found a video that errored! add it to the archive to prevent future errors + if (sub.archive) { + archive_dir = sub.archive; + archive_path = path.join(archive_dir, 'archive.txt') + fs.appendFileSync(archive_path, output['id']); + } + } + } + } catch(e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + } + } + resolve(false); + } else if (output) { + if (output.length === 0 || (output.length === 1 && output[0] === '')) { + logger.verbose('No additional videos to download for ' + sub.name); + resolve(true); + return; + } + for (let i = 0; i < output.length; i++) { + let output_json = null; + try { + output_json = JSON.parse(output[i]); + } catch(e) { + output_json = null; + } + if (!output_json) { + continue; + } + + const reset_videos = i === 0; + handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); + } + + if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { + await setFreshUploads(sub, user_uid); + checkVideosForFreshUploads(sub, user_uid); + } + + resolve(true); + } + }); + }, err => { + logger.error(err); + updateSubscriptionProperty(sub, {downloading: false}, user_uid); + }); +} + +async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) { + // get basePath + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + + let appendedBasePath = getAppendedBasePath(sub, basePath); let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`; - if (sub.custom_output) { + if (desired_path) { + fullOutput = `${desired_path}.%(ext)s`; + } else if (sub.custom_output) { fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`; } - let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json']; + let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json']; let qualityPath = null; if (sub.type && sub.type === 'audio') { @@ -320,7 +400,7 @@ async function getVideosForSub(sub, user_uid = null) { let archive_dir = null; let archive_path = null; - if (useArchive) { + if (useArchive && !redownload) { if (sub.archive) { archive_dir = sub.archive; archive_path = path.join(archive_dir, 'archive.txt') @@ -333,7 +413,7 @@ async function getVideosForSub(sub, user_uid = null) { downloadConfig = ['-f', 'best', '--dump-json']; } - if (sub.timerange) { + if (sub.timerange && !redownload) { downloadConfig.push('--dateafter', sub.timerange); } @@ -350,60 +430,7 @@ async function getVideosForSub(sub, user_uid = null) { downloadConfig.push('--write-thumbnail'); } - // get videos - logger.verbose('Subscription: getting videos for subscription ' + sub.name); - - return new Promise(resolve => { - youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { - logger.verbose('Subscription: finished check for ' + sub.name); - if (err && !output) { - logger.error(err.stderr ? err.stderr : err.message); - if (err.stderr.includes('This video is unavailable')) { - logger.info('An error was encountered with at least one video, backup method will be used.') - try { - const outputs = err.stdout.split(/\r\n|\r|\n/); - for (let i = 0; i < outputs.length; i++) { - const output = JSON.parse(outputs[i]); - handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) - if (err.stderr.includes(output['id']) && archive_path) { - // we found a video that errored! add it to the archive to prevent future errors - fs.appendFileSync(archive_path, output['id']); - } - } - } catch(e) { - logger.error('Backup method failed. See error below:'); - logger.error(e); - } - } - resolve(false); - } else if (output) { - if (output.length === 0 || (output.length === 1 && output[0] === '')) { - logger.verbose('No additional videos to download for ' + sub.name); - resolve(true); - } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - - const reset_videos = i === 0; - handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); - - // TODO: Potentially store downloaded files in db? - - } - resolve(true); - } - }); - }, err => { - logger.error(err); - }); + return downloadConfig; } function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { @@ -418,6 +445,14 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ // add to db sub_db.get('videos').push(output_json).write(); } else { + path_object = path.parse(output_json['_filename']); + const path_string = path.format(path_object); + + if (sub_db.get('videos').find({path: path_string}).value()) { + // file already exists in DB, return early to avoid reseting the download date + return; + } + db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); const url = output_json['webpage_url']; if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 @@ -431,13 +466,28 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ } } -function getAllSubscriptions(user_uid = null) { +function getSubscriptions(user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').value(); else return db.get('subscriptions').value(); } +function getAllSubscriptions() { + let subscriptions = null; + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + if (multiUserMode) { + subscriptions = []; + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); + } + } else { + subscriptions = getSubscriptions(); + } + return subscriptions; +} + function getSubscription(subID, user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); @@ -461,6 +511,21 @@ function updateSubscription(sub, user_uid = null) { return true; } +function updateSubscriptionPropertyMultiple(subs, assignment_obj) { + subs.forEach(sub => { + updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); + }); +} + +function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { + if (user_uid) { + users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); + } else { + db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); + } + return true; +} + function subExists(subID, user_uid = null) { if (user_uid) return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); @@ -468,6 +533,52 @@ function subExists(subID, user_uid = null) { return !!db.get('subscriptions').find({id: subID}).value(); } +async function setFreshUploads(sub, user_uid) { + const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); + sub.videos.forEach(async video => { + if (current_date === video['upload_date'].replace(/-/g, '')) { + // set upload as fresh + const video_uid = video['uid']; + await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']); + } + }); +} + +async function checkVideosForFreshUploads(sub, user_uid) { + const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); + sub.videos.forEach(async video => { + if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) { + checkVideoIfBetterExists(video, sub, user_uid) + } + }); +} + +async function checkVideoIfBetterExists(file_obj, sub, user_uid) { + const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4); + const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path); + logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); + // simulate a download to verify that a better version exists + youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => { + if (err) { + // video is not available anymore for whatever reason + } else if (output) { + const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; + if (output[metric_to_compare] > file_obj[metric_to_compare]) { + // download new video as the simulated one is better + youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => { + if (err) { + logger.verbose(`Failed to download better version of video ${file_obj['id']}`); + } else if (output) { + logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); + await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']); + } + }); + } + } + }); + await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']); +} + // helper functions function getAppendedBasePath(sub, base_path) { @@ -505,6 +616,7 @@ async function removeIDFromArchive(archive_path, id) { module.exports = { getSubscription : getSubscription, getSubscriptionByName : getSubscriptionByName, + getSubscriptions : getSubscriptions, getAllSubscriptions : getAllSubscriptions, updateSubscription : updateSubscription, subscribe : subscribe, @@ -513,5 +625,6 @@ module.exports = { getVideosForSub : getVideosForSub, removeIDFromArchive : removeIDFromArchive, setLogger : setLogger, - initialize : initialize + initialize : initialize, + updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple } diff --git a/backend/utils.js b/backend/utils.js index ed3b584..b18ed6a 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -20,7 +20,7 @@ function getTrueFileName(unfixed_path, type) { return fixed_path; } -async function getDownloadedFilesByType(basePath, type) { +async function getDownloadedFilesByType(basePath, type, full_metadata = false) { // return empty array if the path doesn't exist if (!(await fs.pathExists(basePath))) return []; @@ -36,18 +36,17 @@ async function getDownloadedFilesByType(basePath, type) { var id = file_path.substring(0, file_path.length-4); var jsonobj = await getJSONByType(type, id, basePath); if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; + if (full_metadata) { + jsonobj['id'] = id; + files.push(jsonobj); + continue; + } var upload_date = jsonobj.upload_date; upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - - var size = stats.size; var isaudio = type === 'audio'; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader, + stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr); files.push(file_obj); } return files; @@ -184,7 +183,7 @@ async function recFindByExt(base,ext,files,result) // objects -function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) { +function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { this.id = id; this.title = title; this.thumbnailURL = thumbnailURL; @@ -195,6 +194,10 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p this.size = size; this.path = path; this.upload_date = upload_date; + this.description = description; + this.view_count = view_count; + this.height = height; + this.abr = abr; } module.exports = { diff --git a/package.json b/package.json index e34f8a8..ab15811 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "youtube-dl-material", - "version": "4.1.0", + "version": "4.2.0", "license": "MIT", "scripts": { "ng": "ng", diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 056ee7b..0ae868f 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -32,7 +32,7 @@ import { DragDropModule } from '@angular/cdk/drag-drop'; import { ClipboardModule } from '@angular/cdk/clipboard'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { AppComponent } from './app.component'; -import { HttpClientModule, HttpClient } from '@angular/common/http'; +import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http'; import { PostsService } from 'app/posts.services'; import { FileCardComponent } from './file-card/file-card.component'; import { RouterModule } from '@angular/router'; @@ -84,6 +84,8 @@ import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dia import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component'; import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component'; +import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.component'; +import { H401Interceptor } from './http.interceptor'; registerLocaleData(es, 'es'); @@ -110,6 +112,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible VideoInfoDialogComponent, ArgModifierDialogComponent, HighlightPipe, + LinkifyPipe, UpdaterComponent, UpdateProgressDialogComponent, ShareMediaDialogComponent, @@ -130,7 +133,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible EditSubscriptionDialogComponent, CustomPlaylistsComponent, EditCategoryDialogComponent, - TwitchChatComponent + TwitchChatComponent, + SeeMoreComponent ], imports: [ CommonModule, @@ -188,10 +192,12 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible SettingsComponent ], providers: [ - PostsService + PostsService, + { provide: HTTP_INTERCEPTORS, useClass: H401Interceptor, multi: true } ], exports: [ - HighlightPipe + HighlightPipe, + LinkifyPipe ], bootstrap: [AppComponent] }) diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index d2d65d6..73e3036 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -62,7 +62,7 @@ export class CustomPlaylistsComponent implements OnInit { } else { localStorage.setItem('player_navigator', this.router.url); const fileNames = playlist.fileNames; - this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID}]); + this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID, auto: playlist.auto}]); } } else { // playlist not found diff --git a/src/app/components/login/login.component.ts b/src/app/components/login/login.component.ts index d5c5816..edf7f99 100644 --- a/src/app/components/login/login.component.ts +++ b/src/app/components/login/login.component.ts @@ -27,7 +27,7 @@ export class LoginComponent implements OnInit { constructor(private postsService: PostsService, private snackBar: MatSnackBar, private router: Router) { } ngOnInit(): void { - if (this.postsService.isLoggedIn) { + if (this.postsService.isLoggedIn && localStorage.getItem('jwt_token') !== 'null') { this.router.navigate(['/home']); } this.postsService.service_initialized.subscribe(init => { diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index a0667c7..83bd72f 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -98,7 +98,7 @@ export class RecentVideosComponent implements OnInit { private filterFiles(value: string) { const filterValue = value.toLowerCase(); - this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue)); + this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.name?.toLowerCase().includes(filterValue)); this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}); } @@ -127,9 +127,11 @@ export class RecentVideosComponent implements OnInit { this.normal_files_received = false; this.postsService.getAllFiles().subscribe(res => { this.files = res['files']; - this.files.forEach(file => { + for (let i = 0; i < this.files.length; i++) { + const file = this.files[i]; file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration); - }); + file.index = i; + } this.files.sort(this.sortFiles); if (this.search_mode) { this.filterFiles(this.search_text); @@ -247,7 +249,9 @@ export class RecentVideosComponent implements OnInit { this.postsService.deleteFile(file.uid, file.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { if (result) { this.postsService.openSnackBar('Delete success!', 'OK.'); - this.files.splice(index, 1); + this.files.splice(file.index, 1); + for (let i = 0; i < this.files.length; i++) { this.files[i].index = i } + this.filterByProperty(this.filterProperty['property']); } else { this.postsService.openSnackBar('Delete failed!', 'OK.'); } diff --git a/src/app/components/see-more/see-more.component.html b/src/app/components/see-more/see-more.component.html new file mode 100644 index 0000000..297d820 --- /dev/null +++ b/src/app/components/see-more/see-more.component.html @@ -0,0 +1,11 @@ + + + + + See more. + + + See less. + + + \ No newline at end of file diff --git a/src/app/components/see-more/see-more.component.scss b/src/app/components/see-more/see-more.component.scss new file mode 100644 index 0000000..1843c11 --- /dev/null +++ b/src/app/components/see-more/see-more.component.scss @@ -0,0 +1,7 @@ +.text { + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + word-wrap: break-word; + } \ No newline at end of file diff --git a/src/app/components/see-more/see-more.component.spec.ts b/src/app/components/see-more/see-more.component.spec.ts new file mode 100644 index 0000000..608bee6 --- /dev/null +++ b/src/app/components/see-more/see-more.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { SeeMoreComponent } from './see-more.component'; + +describe('SeeMoreComponent', () => { + let component: SeeMoreComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ SeeMoreComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(SeeMoreComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/see-more/see-more.component.ts b/src/app/components/see-more/see-more.component.ts new file mode 100644 index 0000000..8573cb9 --- /dev/null +++ b/src/app/components/see-more/see-more.component.ts @@ -0,0 +1,60 @@ +import { Component, Input, OnInit, Pipe, PipeTransform } from '@angular/core'; +import { DomSanitizer } from '@angular/platform-browser'; + +@Pipe({ name: 'linkify' }) +export class LinkifyPipe implements PipeTransform { + + constructor(private _domSanitizer: DomSanitizer) {} + + transform(value: any, args?: any): any { + return this._domSanitizer.bypassSecurityTrustHtml(this.stylize(value)); + } + + // Modify this method according to your custom logic + private stylize(text: string): string { + let stylizedText: string = ''; + if (text && text.length > 0) { + for (let line of text.split("\n")) { + for (let t of line.split(" ")) { + if (t.startsWith("http") && t.length>7) { + stylizedText += `${t} `; + } + else + stylizedText += t + " "; + } + stylizedText += '
'; + } + return stylizedText; + } + else return text; + } + +} + +@Component({ + selector: 'app-see-more', + templateUrl: './see-more.component.html', + providers: [LinkifyPipe], + styleUrls: ['./see-more.component.scss'] +}) +export class SeeMoreComponent implements OnInit { + + see_more_active = false; + + @Input() text = ''; + @Input() line_limit = 2; + + constructor() { } + + ngOnInit(): void { + } + + toggleSeeMore() { + this.see_more_active = !this.see_more_active; + } + + parseText() { + return this.text.replace(/(http.*?\s)/, "$1") + } + +} diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index a080324..066d78e 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -1,5 +1,10 @@
-
{{(file_obj.type === 'audio' || file_obj.isAudio) ? 'audiotrack' : 'movie'}}  {{file_obj.registered | date:'shortDate' : undefined : locale.ngID}}
+
+ {{(file_obj.type === 'audio' || file_obj.isAudio) ? 'audiotrack' : 'movie'}} +    + Auto-generated + {{file_obj.registered | date:'shortDate' : undefined : locale.ngID}} +
- + diff --git a/src/app/components/unified-file-card/unified-file-card.component.scss b/src/app/components/unified-file-card/unified-file-card.component.scss index 067ef4a..8d8ea1a 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.scss +++ b/src/app/components/unified-file-card/unified-file-card.component.scss @@ -111,6 +111,11 @@ top: 1px; left: 5px; z-index: 99999; + width: calc(100% - 8px); + white-space: nowrap; + overflow: hidden; + display: block; + text-overflow: ellipsis; } .audio-video-icon { diff --git a/src/app/consts.ts b/src/app/consts.ts index ad0197d..950aff6 100644 --- a/src/app/consts.ts +++ b/src/app/consts.ts @@ -1 +1 @@ -export const CURRENT_VERSION = 'v4.1'; +export const CURRENT_VERSION = 'v4.2'; diff --git a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html index 00d5485..f76af9f 100644 --- a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html +++ b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.html @@ -1,8 +1,11 @@ -

Editing

 {{sub.name}} +

Editing {{sub.name}} (Paused)

+
+ Paused +
Download all uploads
@@ -31,7 +34,7 @@
-
+
Streaming-only mode
diff --git a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts index b1d2dd4..19ff03b 100644 --- a/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts +++ b/src/app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component.ts @@ -61,9 +61,13 @@ export class EditSubscriptionDialogComponent implements OnInit { ]; constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialog: MatDialog, private postsService: PostsService) { - this.sub = this.data.sub; + this.sub = JSON.parse(JSON.stringify(this.data.sub)); this.new_sub = JSON.parse(JSON.stringify(this.sub)); + // ignore videos to keep requests small + delete this.sub['videos']; + delete this.new_sub['videos']; + this.audioOnlyMode = this.sub.type === 'audio'; this.download_all = !this.sub.timerange; diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.html b/src/app/dialogs/modify-playlist/modify-playlist.component.html index d35db91..69f4cad 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.html +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.html @@ -8,14 +8,24 @@
+
+
+ Normal order  + Reverse order  + +
+ +
+ +
+
+ -
{{playlist_item}}
+ +
{{playlist_item}}
- -
- -
+ diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.scss b/src/app/dialogs/modify-playlist/modify-playlist.component.scss index 84d3c54..0be9a78 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.scss +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.scss @@ -30,11 +30,6 @@ border: none; transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); } -.add-content-button { -margin-top: 15px; -margin-bottom: 10px; -} - .remove-item-button { right: 10px; position: absolute; diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.ts b/src/app/dialogs/modify-playlist/modify-playlist.component.ts index 18db662..414fc92 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts @@ -15,6 +15,7 @@ export class ModifyPlaylistComponent implements OnInit { available_files = []; all_files = []; playlist_updated = false; + reverse_order = false; constructor(@Inject(MAT_DIALOG_DATA) public data: any, private postsService: PostsService, @@ -26,6 +27,8 @@ export class ModifyPlaylistComponent implements OnInit { this.original_playlist = JSON.parse(JSON.stringify(this.data.playlist)); this.getFiles(); } + + this.reverse_order = localStorage.getItem('default_playlist_order_reversed') === 'true'; } getFiles() { @@ -72,11 +75,23 @@ export class ModifyPlaylistComponent implements OnInit { } removeContent(index) { + if (this.reverse_order) { + index = this.playlist.fileNames.length - 1 - index; + } this.playlist.fileNames.splice(index, 1); this.processFiles(); } + togglePlaylistOrder() { + this.reverse_order = !this.reverse_order; + localStorage.setItem('default_playlist_order_reversed', '' + this.reverse_order); + } + drop(event: CdkDragDrop) { + if (this.reverse_order) { + event.previousIndex = this.playlist.fileNames.length - 1 - event.previousIndex; + event.currentIndex = this.playlist.fileNames.length - 1 - event.currentIndex; + } moveItemInArray(this.playlist.fileNames, event.previousIndex, event.currentIndex); } diff --git a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html index 1ebff5a..4517a47 100644 --- a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html +++ b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.html @@ -1,4 +1,4 @@ -

{{sub.name}}

+

{{sub.name}} (Paused)

diff --git a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html index 7323385..e62d1c9 100644 --- a/src/app/dialogs/video-info-dialog/video-info-dialog.component.html +++ b/src/app/dialogs/video-info-dialog/video-info-dialog.component.html @@ -25,6 +25,10 @@
Upload Date: 
{{file.upload_date ? file.upload_date : 'N/A'}}
+
+
Category: 
+
{{file.category.name}}N/A
+
diff --git a/src/app/http.interceptor.ts b/src/app/http.interceptor.ts new file mode 100644 index 0000000..edde22b --- /dev/null +++ b/src/app/http.interceptor.ts @@ -0,0 +1,34 @@ +import { HttpEvent, HttpHandler, HttpInterceptor, HttpRequest } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { MatSnackBar } from '@angular/material/snack-bar'; +import { Router } from '@angular/router'; +import { Observable, throwError } from 'rxjs'; +import { catchError } from 'rxjs/operators'; + +@Injectable() +export class H401Interceptor implements HttpInterceptor { + + constructor(private router: Router, private snackBar: MatSnackBar) { } + + intercept(request: HttpRequest, next: HttpHandler): Observable> { + return next.handle(request).pipe(catchError(err => { + if (err.status === 401) { + localStorage.setItem('jwt_token', null); + if (this.router.url !== '/login') { + this.router.navigate(['/login']).then(() => { + this.openSnackBar('Login expired, please login again.'); + }); + } + } + + const error = err.error.message || err.statusText; + return throwError(error); + })); + } + + public openSnackBar(message: string, action: string = '') { + this.snackBar.open(message, action, { + duration: 2000, + }); + } +} \ No newline at end of file diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index 59956c8..9cb905e 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -187,92 +187,3 @@

Custom playlists

- - \ No newline at end of file diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 013d0f3..30694bf 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -8,7 +8,6 @@ import { MatSnackBar } from '@angular/material/snack-bar'; import { saveAs } from 'file-saver'; import { YoutubeSearchService, Result } from '../youtube-search.service'; import { Router, ActivatedRoute } from '@angular/router'; -import { CreatePlaylistComponent } from 'app/create-playlist/create-playlist.component'; import { Platform } from '@angular/cdk/platform'; import { v4 as uuid } from 'uuid'; import { ArgModifierDialogComponent } from 'app/dialogs/arg-modifier-dialog/arg-modifier-dialog.component'; @@ -244,13 +243,6 @@ export class MainComponent implements OnInit { this.useDefaultDownloadingAgent = this.postsService.config['Advanced']['use_default_downloading_agent']; this.customDownloadingAgent = this.postsService.config['Advanced']['custom_downloading_agent']; - - - if (this.fileManagerEnabled) { - this.getMp3s(); - this.getMp4s(); - } - if (this.youtubeSearchEnabled && this.youtubeAPIKey) { this.youtubeSearch.initializeAPI(this.youtubeAPIKey); this.attachToInput(); @@ -335,61 +327,6 @@ export class MainComponent implements OnInit { this.setCols(); } - // file manager stuff - - getMp3s() { - this.postsService.getMp3s().subscribe(result => { - const mp3s = result['mp3s']; - const playlists = result['playlists']; - // if they are different - if (JSON.stringify(this.mp3s) !== JSON.stringify(mp3s)) { this.mp3s = mp3s }; - this.playlists.audio = playlists; - - // get thumbnail url by using first video. this is a temporary hack - for (let i = 0; i < this.playlists.audio.length; i++) { - const playlist = this.playlists.audio[i]; - let videoToExtractThumbnail = null; - for (let j = 0; j < this.mp3s.length; j++) { - if (this.mp3s[j].id === playlist.fileNames[0]) { - // found the corresponding file - videoToExtractThumbnail = this.mp3s[j]; - } - } - - if (videoToExtractThumbnail) { this.playlist_thumbnails[playlist.id] = videoToExtractThumbnail.thumbnailURL; } - } - }, error => { - console.log(error); - }); - } - - getMp4s() { - this.postsService.getMp4s().subscribe(result => { - const mp4s = result['mp4s']; - const playlists = result['playlists']; - // if they are different - if (JSON.stringify(this.mp4s) !== JSON.stringify(mp4s)) { this.mp4s = mp4s }; - this.playlists.video = playlists; - - // get thumbnail url by using first video. this is a temporary hack - for (let i = 0; i < this.playlists.video.length; i++) { - const playlist = this.playlists.video[i]; - let videoToExtractThumbnail = null; - for (let j = 0; j < this.mp4s.length; j++) { - if (this.mp4s[j].id === playlist.fileNames[0]) { - // found the corresponding file - videoToExtractThumbnail = this.mp4s[j]; - } - } - - if (videoToExtractThumbnail) { this.playlist_thumbnails[playlist.id] = videoToExtractThumbnail.thumbnailURL; } - } - }, - error => { - console.log(error); - }); - } - public setCols() { if (window.innerWidth <= 350) { this.files_cols = 1; @@ -437,44 +374,6 @@ export class MainComponent implements OnInit { return null; } - public removeFromMp3(name: string) { - for (let i = 0; i < this.mp3s.length; i++) { - if (this.mp3s[i].id === name || this.mp3s[i].id + '.mp3' === name) { - this.mp3s.splice(i, 1); - } - } - this.getMp3s(); - } - - public removePlaylistMp3(playlistID, index) { - this.postsService.removePlaylist(playlistID, 'audio').subscribe(res => { - if (res['success']) { - this.playlists.audio.splice(index, 1); - this.openSnackBar('Playlist successfully removed.', ''); - } - this.getMp3s(); - }); - } - - public removeFromMp4(name: string) { - for (let i = 0; i < this.mp4s.length; i++) { - if (this.mp4s[i].id === name || this.mp4s[i].id + '.mp4' === name) { - this.mp4s.splice(i, 1); - } - } - this.getMp4s(); - } - - public removePlaylistMp4(playlistID, index) { - this.postsService.removePlaylist(playlistID, 'video').subscribe(res => { - if (res['success']) { - this.playlists.video.splice(index, 1); - this.openSnackBar('Playlist successfully removed.', ''); - } - this.getMp4s(); - }); - } - // download helpers downloadHelperMp3(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { @@ -504,16 +403,6 @@ export class MainComponent implements OnInit { // remove download from current downloads this.removeDownloadFromCurrentDownloads(new_download); - - // reloads mp3s - if (this.fileManagerEnabled) { - this.getMp3s(); - setTimeout(() => { - this.audioFileCards.forEach(filecard => { - filecard.onHoverResponse(); - }); - }, 200); - } } downloadHelperMp4(name, uid, is_playlist = false, forceView = false, new_download = null, navigate_mode = false) { @@ -543,16 +432,6 @@ export class MainComponent implements OnInit { // remove download from current downloads this.removeDownloadFromCurrentDownloads(new_download); - - // reloads mp4s - if (this.fileManagerEnabled) { - this.getMp4s(); - setTimeout(() => { - this.videoFileCards.forEach(filecard => { - filecard.onHoverResponse(); - }); - }, 200); - } } // download click handler @@ -745,8 +624,6 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded this.postsService.deleteFile(name, 'video').subscribe(delRes => { - // reload mp3s - this.getMp3s(); }); } }); @@ -762,8 +639,6 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded this.postsService.deleteFile(name, 'audio').subscribe(delRes => { - // reload mp4s - this.getMp4s(); }); } }); @@ -1110,25 +985,6 @@ export class MainComponent implements OnInit { } } - // creating a playlist - openCreatePlaylistDialog(type) { - const dialogRef = this.dialog.open(CreatePlaylistComponent, { - data: { - filesToSelectFrom: (type === 'audio') ? this.mp3s : this.mp4s, - type: type - } - }); - dialogRef.afterClosed().subscribe(result => { - if (result) { - if (type === 'audio') { this.getMp3s() }; - if (type === 'video') { this.getMp4s() }; - this.openSnackBar('Successfully created playlist!', ''); - } else if (result === false) { - this.openSnackBar('ERROR: failed to create playlist!', ''); - } - }); - } - // modify custom args openArgsModifierDialog() { const dialogRef = this.dialog.open(ArgModifierDialogComponent, { diff --git a/src/app/player/player.component.css b/src/app/player/player.component.css index 42247ef..202b80d 100644 --- a/src/app/player/player.component.css +++ b/src/app/player/player.component.css @@ -37,10 +37,8 @@ } .spinner { - width: 50px; - height: 50px; - bottom: 3px; - left: 3px; + bottom: 1px; + left: 2px; position: absolute; } diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 53fad9b..6aaa7b0 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -8,20 +8,49 @@
+
+
+
+
+ {{db_file['local_view_count'] ? db_file['local_view_count']+1 : 1}} views +
+
+ +

+ +

+
+ +

+ No description available. +

+
+
+
+ + + + + + + + + + +
+
+
+
{{playlist_item.label}}
- + - -
- -
@@ -30,16 +59,6 @@
- -
- - - -
-
- - -
diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 412a583..123e2b4 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -160,6 +160,10 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { this.openSnackBar('Failed to get file information from the server.', 'Dismiss'); return; } + this.postsService.incrementViewCount(this.db_file['uid'], null, this.uuid).subscribe(res => {}, err => { + console.error('Failed to increment view count'); + console.error(err); + }); this.sharingEnabled = this.db_file.sharingEnabled; if (!this.fileNames) { // means it's a shared video @@ -186,6 +190,10 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { subscription.videos.forEach(video => { if (video['id'] === this.fileNames[0]) { this.db_file = video; + this.postsService.incrementViewCount(this.db_file['uid'], this.subscription['id'], this.uuid).subscribe(res => {}, err => { + console.error('Failed to increment view count'); + console.error(err); + }); this.show_player = true; this.parseFileNames(); } @@ -199,6 +207,10 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { } getPlaylistFiles() { + if (this.route.snapshot.paramMap.get('auto') === 'true') { + this.show_player = true; + return; + } this.postsService.getPlaylist(this.id, null, this.uuid).subscribe(res => { if (res['playlist']) { this.db_playlist = res['playlist']; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index c3934c7..4ae90dd 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -286,6 +286,10 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'enableSharing', {uid: uid, type: type, is_playlist: is_playlist}, this.httpOptions); } + incrementViewCount(file_uid, sub_id, uuid) { + return this.http.post(this.path + 'incrementViewCount', {file_uid: file_uid, sub_id: sub_id, uuid: uuid}, this.httpOptions); + } + disableSharing(uid, type, is_playlist) { return this.http.post(this.path + 'disableSharing', {uid: uid, type: type, is_playlist: is_playlist}, this.httpOptions); } @@ -370,7 +374,7 @@ export class PostsService implements CanActivate { } getAllSubscriptions() { - return this.http.post(this.path + 'getAllSubscriptions', {}, this.httpOptions); + return this.http.post(this.path + 'getSubscriptions', {}, this.httpOptions); } // current downloads diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 08d4970..bd085b1 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -53,12 +53,15 @@ Base path for videos from your subscribed channels and playlists. It is relative to YTDL-Material's root folder.
-
+
Unit is seconds, only include numbers.
+
+ Redownload fresh uploads +
diff --git a/src/app/subscription/subscription/subscription.component.html b/src/app/subscription/subscription/subscription.component.html index 0d8c917..82747ac 100644 --- a/src/app/subscription/subscription/subscription.component.html +++ b/src/app/subscription/subscription/subscription.component.html @@ -2,8 +2,9 @@

- {{subscription.name}} + {{subscription.name}} (Paused)

+

diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index df0561d..461dcfc 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnDestroy, OnInit } from '@angular/core'; import { PostsService } from 'app/posts.services'; import { ActivatedRoute, Router, ParamMap } from '@angular/router'; import { MatDialog } from '@angular/material/dialog'; @@ -9,7 +9,7 @@ import { EditSubscriptionDialogComponent } from 'app/dialogs/edit-subscription-d templateUrl: './subscription.component.html', styleUrls: ['./subscription.component.scss'] }) -export class SubscriptionComponent implements OnInit { +export class SubscriptionComponent implements OnInit, OnDestroy { id = null; subscription = null; @@ -44,22 +44,11 @@ export class SubscriptionComponent implements OnInit { }; filterProperty = this.filterProperties['upload_date']; downloading = false; - - initialized = false; + sub_interval = null; constructor(private postsService: PostsService, private route: ActivatedRoute, private router: Router, private dialog: MatDialog) { } ngOnInit() { - this.route.paramMap.subscribe((params: ParamMap) => { - this.id = params.get('id'); - this.postsService.service_initialized.subscribe(init => { - if (init) { - this.initialized = true; - this.getConfig(); - this.getSubscription(); - } - }); - }); if (this.route.snapshot.paramMap.get('id')) { this.id = this.route.snapshot.paramMap.get('id'); @@ -67,6 +56,7 @@ export class SubscriptionComponent implements OnInit { if (init) { this.getConfig(); this.getSubscription(); + this.sub_interval = setInterval(() => this.getSubscription(true), 1000); } }); } @@ -78,12 +68,25 @@ export class SubscriptionComponent implements OnInit { } } + ngOnDestroy() { + // prevents subscription getter from running in the background + if (this.sub_interval) { + clearInterval(this.sub_interval); + } + } + goBack() { this.router.navigate(['/subscriptions']); } - getSubscription() { + getSubscription(low_cost = false) { this.postsService.getSubscription(this.id).subscribe(res => { + if (low_cost && res['subscription'].videos.length === this.subscription?.videos.length) { + if (res['subscription']['downloading'] !== this.subscription['downloading']) { + this.subscription['downloading'] = res['subscription']['downloading']; + } + return; + } this.subscription = res['subscription']; this.files = res['files']; if (this.search_mode) { diff --git a/src/app/subscriptions/subscriptions.component.html b/src/app/subscriptions/subscriptions.component.html index 5bf76bb..1c8e838 100644 --- a/src/app/subscriptions/subscriptions.component.html +++ b/src/app/subscriptions/subscriptions.component.html @@ -9,7 +9,7 @@ - {{ sub.name }} + {{ sub.name }} (Paused)
Name not available. Channel retrieval in progress.
@@ -28,7 +28,7 @@
- {{ sub.name }} + {{ sub.name }} (Paused)
Name not available. Playlist retrieval in progress.
diff --git a/src/assets/default.json b/src/assets/default.json index 893896f..532b32e 100644 --- a/src/assets/default.json +++ b/src/assets/default.json @@ -26,7 +26,9 @@ "use_API_key": false, "API_key": "", "use_youtube_API": false, - "youtube_API_key": "" + "youtube_API_key": "", + "use_twitch_API": false, + "twitch_API_key": "" }, "Themes": { "default_theme": "default", @@ -57,7 +59,8 @@ "allow_advanced_download": true, "jwt_expiration": 86400, "logger_level": "debug", - "use_cookies": false + "use_cookies": false, + "default_downloader": "youtube-dlc" } } } \ No newline at end of file