TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-08 15:21:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Changed DB structure again

Browse Source 
Added support for MongoDB

Added tests relating to new DB system

Category rules are now case insensitive

Fixed playlist modification change state
This commit is contained in:
Isaac Abadi
2021-07-16 00:05:08 -06:00
parent 984e990103
commit bb6503e86d
27 changed files with 1426 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
backend/app.js
View File

@@ -79,9 +79,9 @@ const logger = winston.createLogger({
}); });
config_api.initialize(logger); config_api.initialize(logger);
auth_api.initialize(db, users_db, logger);
db_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api); auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
categories_api.initialize(db, users_db, logger, db_api); categories_api.initialize(db, users_db, logger, db_api);
// Set some defaults // Set some defaults
@@ -183,7 +183,7 @@ if (writeConfigMode) {
loadConfig(); loadConfig();
} }
var downloads = {}; var downloads = [];
app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json()); app.use(bodyParser.json());
@@ -210,10 +210,12 @@ async function checkMigrations() {
if (!simplified_db_migration_complete) { if (!simplified_db_migration_complete) {
logger.info('Beginning migration: 4.1->4.2+') logger.info('Beginning migration: 4.1->4.2+')
let success = await simplifyDBFileStructure(); let success = await simplifyDBFileStructure();
success = success && await addMetadataPropertyToDB('view_count'); success = success && await db_api.addMetadataPropertyToDB('view_count');
success = success && await addMetadataPropertyToDB('description'); success = success && await db_api.addMetadataPropertyToDB('description');
success = success && await addMetadataPropertyToDB('height'); success = success && await db_api.addMetadataPropertyToDB('height');
success = success && await addMetadataPropertyToDB('abr'); success = success && await db_api.addMetadataPropertyToDB('abr');
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
if (success) { logger.info('4.1->4.2+ migration complete!'); } if (success) { logger.info('4.1->4.2+ migration complete!'); }
else { logger.error('Migration failed: 4.1->4.2+'); } else { logger.error('Migration failed: 4.1->4.2+'); }
} }
@@ -290,28 +292,6 @@ async function simplifyDBFileStructure() {
return true; return true;
} }
async function addMetadataPropertyToDB(property_key) {
try {
const dirs_to_check = db_api.getFileDirectoriesAndDBs();
for (const dir_to_check of dirs_to_check) {
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (const file of files) {
if (file[property_key]) {
dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write();
}
}
}
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
return true;
} catch(err) {
logger.error(err);
return false;
}
}
async function startServer() { async function startServer() {
if (process.env.USING_HEROKU && process.env.PORT) { if (process.env.USING_HEROKU && process.env.PORT) {
// default to heroku port if using heroku // default to heroku port if using heroku
@@ -612,6 +592,9 @@ async function setConfigFromEnv() {
async function loadConfig() { async function loadConfig() {
loadConfigValues(); loadConfigValues();
// connect to DB
await db_api.connectToDB();
// creates archive path if missing // creates archive path if missing
await fs.ensureDir(archivePath); await fs.ensureDir(archivePath);
@@ -624,7 +607,7 @@ async function loadConfig() {
// get subscriptions // get subscriptions
if (allowSubscriptions) { if (allowSubscriptions) {
// set downloading to false // set downloading to false
let subscriptions = subscriptions_api.getAllSubscriptions(); let subscriptions = await subscriptions_api.getAllSubscriptions();
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
// runs initially, then runs every ${subscriptionCheckInterval} seconds // runs initially, then runs every ${subscriptionCheckInterval} seconds
watchSubscriptions(); watchSubscriptions();
@@ -633,10 +616,10 @@ async function loadConfig() {
}, subscriptionsCheckInterval * 1000); }, subscriptionsCheckInterval * 1000);
} }
db_api.importUnregisteredFiles(); await db_api.importUnregisteredFiles();
// load in previous downloads // load in previous downloads
downloads = db.get('downloads').value(); downloads = await db_api.getRecords('downloads');
// start the server here // start the server here
startServer(); startServer();
@@ -684,7 +667,7 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
} }
async function watchSubscriptions() { async function watchSubscriptions() {
let subscriptions = subscriptions_api.getAllSubscriptions(); let subscriptions = await subscriptions_api.getAllSubscriptions();
if (!subscriptions) return; if (!subscriptions) return;
@@ -903,8 +886,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
// adds download to download helper // adds download to download helper
const download_uid = uuid(); const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared'; const session = sessionID ? sessionID : 'undeclared';
if (!downloads[session]) downloads[session] = {}; let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
downloads[session][download_uid] = { if (!session_downloads) {
session_downloads = {session_id: session};
downloads.push(session_downloads);
}
session_downloads[download_uid] = {
uid: download_uid, uid: download_uid,
ui_uid: options.ui_uid, ui_uid: options.ui_uid,
downloading: true, downloading: true,
@@ -916,7 +903,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
timestamp_start: Date.now(), timestamp_start: Date.now(),
filesize: null filesize: null
}; };
const download = downloads[session][download_uid]; const download = session_downloads[download_uid];
updateDownloads(); updateDownloads();
let download_checker = null; let download_checker = null;
@@ -1027,7 +1014,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
} }
// registers file in DB // registers file in DB
const file_obj = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings); const file_obj = await db_api.registerFileDB2(full_file_path, options.user, category, null, options.cropFileSettings);
// TODO: remove the following line // TODO: remove the following line
if (file_name) file_names.push(file_name); if (file_name) file_names.push(file_name);
@@ -1070,146 +1057,6 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
}); });
} }
async function downloadFileByURL_normal(url, type, options, sessionID = null) {
return new Promise(async resolve => {
var date = Date.now();
var file_uid = null;
const is_audio = type === 'audio';
const ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; }
// prepend with user if needed
let multiUserMode = null;
if (options.user) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, options.user, type);
fs.ensureDirSync(user_path);
fileFolderPath = user_path + path.sep;
multiUserMode = {
user: options.user,
file_path: fileFolderPath
}
options.customFileFolderPath = fileFolderPath;
}
options.downloading_method = 'normal';
const downloadConfig = await generateArgs(url, type, options);
// adds download to download helper
const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared';
if (!downloads[session]) downloads[session] = {};
downloads[session][download_uid] = {
uid: download_uid,
ui_uid: options.ui_uid,
downloading: true,
complete: false,
url: url,
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now()
};
const download = downloads[session][download_uid];
updateDownloads();
const video = youtubedl(url,
// Optional arguments passed to youtube-dl.
downloadConfig,
// Additional options can be given for calling `child_process.execFile()`.
{ cwd: __dirname });
let video_info = null;
let file_size = 0;
// Will be called when the download starts.
video.on('info', function(info) {
video_info = info;
file_size = video_info.size;
const json_path = utils.removeFileExtension(video_info._filename) + '.info.json';
fs.ensureFileSync(json_path);
fs.writeJSONSync(json_path, video_info);
video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' }))
});
// Will be called if download was already completed and there is nothing more to download.
video.on('complete', function complete(info) {
'use strict'
logger.info('file ' + info._filename + ' already downloaded.')
})
let download_pos = 0;
video.on('data', function data(chunk) {
download_pos += chunk.length
// `size` should not be 0 here.
if (file_size) {
let percent = (download_pos / file_size * 100).toFixed(2)
download['percent_complete'] = percent;
}
});
video.on('end', async function() {
let new_date = Date.now();
let difference = (new_date - date)/1000;
logger.debug(`Video download delay: ${difference} seconds.`);
download['timestamp_end'] = Date.now();
download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext];
download['complete'] = true;
updateDownloads();
// audio-only cleanup
if (is_audio) {
// filename fix
video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3';
// ID3 tagging
let tags = {
title: video_info['title'],
artist: video_info['artist'] ? video_info['artist'] : video_info['uploader']
}
let success = NodeID3.write(tags, video_info._filename);
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename);
const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm';
const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4';
// check if audio file is webm
if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']);
else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']);
}
// registers file in DB
const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length);
file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode);
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name));
resolve({
encodedPath: videopathEncoded,
file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready
uid: file_uid
});
});
video.on('error', function error(err) {
logger.error(err);
download[error] = err;
updateDownloads();
resolve(false);
});
});
}
async function generateArgs(url, type, options) { async function generateArgs(url, type, options) {
var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
var globalArgs = config_api.getConfigItem('ytdl_custom_args'); var globalArgs = config_api.getConfigItem('ytdl_custom_args');
@@ -1250,8 +1097,8 @@ async function generateArgs(url, type, options) {
qualityPath = ['-f', customQualityConfiguration]; qualityPath = ['-f', customQualityConfiguration];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) { } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (maxBitrate && is_audio) { } else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate] qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
} }
if (customOutput) { if (customOutput) {
@@ -1401,23 +1248,6 @@ async function getUrlInfos(urls) {
// ffmpeg helper functions // ffmpeg helper functions
async function convertFileToMp3(input_file, output_file) {
logger.verbose(`Converting ${input_file} to ${output_file}...`);
return new Promise(resolve => {
ffmpeg(input_file).noVideo().toFormat('mp3')
.on('end', () => {
logger.verbose(`Conversion for '${output_file}' complete.`);
fs.unlinkSync(input_file)
resolve(true);
})
.on('error', (err) => {
logger.error('Failed to convert audio file to the correct format.');
logger.error(err);
resolve(false);
}).save(output_file);
});
}
async function cropFile(file_path, start, end, ext) { async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => { return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`; const temp_file_path = `${file_path}.cropped${ext}`;
@@ -1456,8 +1286,9 @@ async function writeToBlacklist(type, line) {
// download management functions // download management functions
function updateDownloads() { async function updateDownloads() {
db.assign({downloads: downloads}).write(); await db_api.removeAllRecords('downloads');
if (downloads.length !== 0) await db_api.insertRecordsIntoTable('downloads', downloads);
} }
function checkDownloadPercent(download) { function checkDownloadPercent(download) {
@@ -1489,7 +1320,6 @@ function checkDownloadPercent(download) {
} }
}); });
download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2); download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
updateDownloads();
}); });
} }
@@ -1715,6 +1545,37 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
res.send({success: true}); res.send({success: true});
}); });
app.post('/api/getDBInfo', optionalJwt, async (req, res) => {
const db_info = await db_api.getDBStats();
res.send({db_info: db_info});
});
app.post('/api/transferDB', optionalJwt, async (req, res) => {
const local_to_remote = req.body.local_to_remote;
let success = null;
let error = '';
if (local_to_remote === config_api.getConfigItem('ytdl_use_local_db')) {
success = await db_api.transferDB(local_to_remote);
if (!success) error = 'Unknown error';
else config_api.setConfigItem('ytdl_use_local_db', !local_to_remote);
} else {
success = false;
error = `Failed to transfer DB as it cannot transition into its current status: ${local_to_remote ? 'MongoDB' : 'Local DB'}`;
logger.error(error);
}
res.send({success: success, error: error});
});
app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
let success = null;
let error = '';
success = await db_api.connectToDB(5, true);
if (!success) error = 'Connection string failed.';
res.send({success: success, error: error});
});
app.post('/api/downloadFile', optionalJwt, async function(req, res) { app.post('/api/downloadFile', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos req.setTimeout(0); // remove timeout in case of long videos
const url = req.body.url; const url = req.body.url;
@@ -1731,15 +1592,7 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
cropFileSettings: req.body.cropFileSettings cropFileSettings: req.body.cropFileSettings
} }
const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload(); let result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID);
if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.');
const is_playlist = url.includes('playlist');
let result_obj = null;
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
if (result_obj) { if (result_obj) {
res.send(result_obj); res.send(result_obj);
} else { } else {
@@ -1752,29 +1605,17 @@ app.post('/api/killAllDownloads', optionalJwt, async function(req, res) {
res.send(result_obj); res.send(result_obj);
}); });
/**
* add thumbnails if present
* @param files - List of files with thumbnailPath property.
*/
async function addThumbnails(files) {
await Promise.all(files.map(async file => {
const thumbnailPath = file['thumbnailPath'];
if (thumbnailPath && (await fs.pathExists(thumbnailPath))) {
file['thumbnailBlob'] = await fs.readFile(thumbnailPath);
}
}));
}
// gets all download mp3s // gets all download mp3s
app.get('/api/getMp3s', optionalJwt, async function(req, res) { app.get('/api/getMp3s', optionalJwt, async function(req, res) {
var mp3s = db.get('files').value().filter(file => file.isAudio === true); // TODO: simplify
var playlists = db.get('playlists').value(); let mp3s = await db_api.getRecords('files', {isAudio: true});
let playlists = await db_api.getRecords('playlists');
const is_authenticated = req.isAuthenticated(); const is_authenticated = req.isAuthenticated();
if (is_authenticated) { if (is_authenticated) {
// get user audio files/playlists // get user audio files/playlists
auth_api.passport.authenticate('jwt') auth_api.passport.authenticate('jwt')
mp3s = auth_api.getUserVideos(req.user.uid, 'audio'); mp3s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: true});
playlists = auth_api.getUserPlaylists(req.user.uid); playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
} }
mp3s = JSON.parse(JSON.stringify(mp3s)); mp3s = JSON.parse(JSON.stringify(mp3s));
@@ -1787,15 +1628,15 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) {
// gets all download mp4s // gets all download mp4s
app.get('/api/getMp4s', optionalJwt, async function(req, res) { app.get('/api/getMp4s', optionalJwt, async function(req, res) {
var mp4s = db.get('files').value().filter(file => file.isAudio === false); let mp4s = await db_api.getRecords('files', {isAudio: false});
var playlists = db.get('playlists').value(); let playlists = await db_api.getRecords('playlists');
const is_authenticated = req.isAuthenticated(); const is_authenticated = req.isAuthenticated();
if (is_authenticated) { if (is_authenticated) {
// get user videos/playlists // get user videos/playlists
auth_api.passport.authenticate('jwt') auth_api.passport.authenticate('jwt')
mp4s = auth_api.getUserVideos(req.user.uid, 'video'); mp4s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: false});
playlists = auth_api.getUserPlaylists(req.user.uid); playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
} }
mp4s = JSON.parse(JSON.stringify(mp4s)); mp4s = JSON.parse(JSON.stringify(mp4s));
@@ -1806,20 +1647,14 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) {
}); });
}); });
app.post('/api/getFile', optionalJwt, function (req, res) { app.post('/api/getFile', optionalJwt, async function (req, res) {
var uid = req.body.uid; var uid = req.body.uid;
var type = req.body.type; var type = req.body.type;
var uuid = req.body.uuid; var uuid = req.body.uuid;
var file = null; var file = await db_api.getRecord('files', {uid: uid});
if (req.isAuthenticated()) { if (uuid && !file['sharingEnabled']) file = null;
file = auth_api.getUserVideo(req.user.uid, uid);
} else if (uuid) {
file = auth_api.getUserVideo(uuid, uid, true);
} else {
file = db.get('files').find({uid: uid}).value();
}
// check if chat exists for twitch videos // check if chat exists for twitch videos
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json'); if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
@@ -1842,18 +1677,12 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
let playlists = null; let playlists = null;
const uuid = req.isAuthenticated() ? req.user.uid : null; const uuid = req.isAuthenticated() ? req.user.uid : null;
let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : []; let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (await subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : [];
// get basic info depending on multi-user mode being enabled files = await db_api.getRecords('files', {user_uid: uuid});
if (uuid) { playlists = await db_api.getRecords('playlists', {user_uid: uuid});
files = auth_api.getUserVideos(req.user.uid);
playlists = auth_api.getUserPlaylists(req.user.uid, files);
} else {
files = db.get('files').value();
playlists = JSON.parse(JSON.stringify(db.get('playlists').value()));
}
const categories = categories_api.getCategoriesAsPlaylists(files); const categories = await categories_api.getCategoriesAsPlaylists(files);
if (categories) { if (categories) {
playlists = playlists.concat(categories); playlists = playlists.concat(categories);
} }
@@ -1872,11 +1701,6 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
files = JSON.parse(JSON.stringify(files)); files = JSON.parse(JSON.stringify(files));
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
// add thumbnails if present
// await addThumbnails(files);
}
res.send({ res.send({
files: files, files: files,
playlists: playlists playlists: playlists
@@ -1952,7 +1776,7 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
}); });
// video sharing // video sharing
app.post('/api/enableSharing', optionalJwt, function(req, res) { app.post('/api/enableSharing', optionalJwt, async (req, res) => {
var uid = req.body.uid; var uid = req.body.uid;
var is_playlist = req.body.is_playlist; var is_playlist = req.body.is_playlist;
let success = false; let success = false;
@@ -1968,15 +1792,9 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) {
try { try {
success = true; success = true;
if (!is_playlist) { if (!is_playlist) {
db.get(`files`) await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: true})
.find({uid: uid})
.assign({sharingEnabled: true})
.write();
} else if (is_playlist) { } else if (is_playlist) {
db.get(`playlists`) await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true});
.find({id: uid})
.assign({sharingEnabled: true})
.write();
} else if (false) { } else if (false) {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory. // time they are requested from the subscription directory.
@@ -1995,7 +1813,7 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) {
}); });
}); });
app.post('/api/disableSharing', optionalJwt, function(req, res) { app.post('/api/disableSharing', optionalJwt, async function(req, res) {
var type = req.body.type; var type = req.body.type;
var uid = req.body.uid; var uid = req.body.uid;
var is_playlist = req.body.is_playlist; var is_playlist = req.body.is_playlist;
@@ -2012,15 +1830,9 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) {
try { try {
success = true; success = true;
if (!is_playlist && type !== 'subscription') { if (!is_playlist && type !== 'subscription') {
db.get(`files`) await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
.find({uid: uid})
.assign({sharingEnabled: false})
.write();
} else if (is_playlist) { } else if (is_playlist) {
db.get(`playlists`) await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
.find({id: uid})
.assign({sharingEnabled: false})
.write();
} else if (type === 'subscription') { } else if (type === 'subscription') {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory. // time they are requested from the subscription directory.
@@ -2062,7 +1874,7 @@ app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
// categories // categories
app.post('/api/getAllCategories', optionalJwt, async (req, res) => { app.post('/api/getAllCategories', optionalJwt, async (req, res) => {
const categories = db.get('categories').value(); const categories = await db_api.getRecords('categories');
res.send({categories: categories}); res.send({categories: categories});
}); });
@@ -2075,7 +1887,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => {
custom_output: '' custom_output: ''
}; };
db.get('categories').push(new_category).write(); await db_api.insertRecordIntoTable('categories', new_category);
res.send({ res.send({
new_category: new_category, new_category: new_category,
@@ -2086,7 +1898,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => {
app.post('/api/deleteCategory', optionalJwt, async (req, res) => { app.post('/api/deleteCategory', optionalJwt, async (req, res) => {
const category_uid = req.body.category_uid; const category_uid = req.body.category_uid;
db.get('categories').remove({uid: category_uid}).write(); await db_api.removeRecord('categories', {uid: category_uid});
res.send({ res.send({
success: true success: true
@@ -2095,13 +1907,14 @@ app.post('/api/deleteCategory', optionalJwt, async (req, res) => {
app.post('/api/updateCategory', optionalJwt, async (req, res) => { app.post('/api/updateCategory', optionalJwt, async (req, res) => {
const category = req.body.category; const category = req.body.category;
db.get('categories').find({uid: category.uid}).assign(category).write(); await db_api.updateRecord('categories', {uid: category.uid}, category)
res.send({success: true}); res.send({success: true});
}); });
app.post('/api/updateCategories', optionalJwt, async (req, res) => { app.post('/api/updateCategories', optionalJwt, async (req, res) => {
const categories = req.body.categories; const categories = req.body.categories;
db.get('categories').assign(categories).write(); await db_api.removeAllRecords('categories');
await db_api.insertRecordsIntoTable('categories', categories);
res.send({success: true}); res.send({success: true});
}); });
@@ -2200,9 +2013,9 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
// get sub from db // get sub from db
let subscription = null; let subscription = null;
if (subID) { if (subID) {
subscription = subscriptions_api.getSubscription(subID, user_uid) subscription = await subscriptions_api.getSubscription(subID, user_uid)
} else if (subName) { } else if (subName) {
subscription = subscriptions_api.getSubscriptionByName(subName, user_uid) subscription = await subscriptions_api.getSubscriptionByName(subName, user_uid)
} }
if (!subscription) { if (!subscription) {
@@ -2213,7 +2026,8 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
// get sub videos // get sub videos
if (subscription.name && !subscription.streamingOnly) { if (subscription.name && !subscription.streamingOnly) {
var parsed_files = subscription.videos; var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
subscription['videos'] = parsed_files;
if (!parsed_files) { if (!parsed_files) {
parsed_files = []; parsed_files = [];
let base_path = null; let base_path = null;
@@ -2310,7 +2124,7 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
let user_uid = req.isAuthenticated() ? req.user.uid : null; let user_uid = req.isAuthenticated() ? req.user.uid : null;
// get subs from api // get subs from api
let subscriptions = subscriptions_api.getSubscriptions(user_uid); let subscriptions = await subscriptions_api.getSubscriptions(user_uid);
res.send({ res.send({
subscriptions: subscriptions subscriptions: subscriptions
@@ -2343,7 +2157,8 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
for (let i = 0; i < playlist['uids'].length; i++) { for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i]; const uid = playlist['uids'][i];
const file_obj = await db_api.getVideo(uid, uuid); const file_obj = await db_api.getVideo(uid, uuid);
file_objs.push(file_obj); if (file_obj) file_objs.push(file_obj);
// TODO: remove file from playlist if could not be found
} }
} }
@@ -2364,10 +2179,7 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => {
if (req.isAuthenticated()) { if (req.isAuthenticated()) {
auth_api.updatePlaylistFiles(req.user.uid, playlistID, uids); auth_api.updatePlaylistFiles(req.user.uid, playlistID, uids);
} else { } else {
db.get(`playlists`) await db_api.updateRecord('playlists', {id: playlistID}, {uids: uids})
.find({id: playlistID})
.assign({uids: uids})
.write();
} }
success = true; success = true;
@@ -2393,14 +2205,8 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
let success = null; let success = null;
try { try {
if (req.isAuthenticated()) { // removes playlist from playlists
auth_api.removePlaylist(req.user.uid, playlistID); await db_api.removeRecord('playlists', {id: playlistID})
} else {
// removes playlist from playlists
db.get(`playlists`)
.remove({id: playlistID})
.write();
}
success = true; success = true;
} catch(e) { } catch(e) {
@@ -2611,20 +2417,14 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
}); });
app.post('/api/download', async (req, res) => { app.post('/api/download', async (req, res) => {
var session_id = req.body.session_id; const session_id = req.body.session_id;
var download_id = req.body.download_id; const download_id = req.body.download_id;
const session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session_id);
let found_download = null; let found_download = null;
// find download // find download
if (downloads[session_id] && Object.keys(downloads[session_id])) { if (session_downloads && Object.keys(session_downloads)) {
let session_downloads = Object.values(downloads[session_id]); found_download = Object.values(session_downloads).find(session_download => session_download['ui_uid'] === download_id);
for (let i = 0; i < session_downloads.length; i++) {
let session_download = session_downloads[i];
if (session_download && session_download['ui_uid'] === download_id) {
found_download = session_download;
break;
}
}
} }
if (found_download) { if (found_download) {
@@ -2642,26 +2442,22 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
var download_id = req.body.download_id; var download_id = req.body.download_id;
if (delete_all) { if (delete_all) {
// delete all downloads // delete all downloads
downloads = {}; downloads = [];
success = true; success = true;
} else if (download_id) { } else if (download_id) {
// delete just 1 download // delete just 1 download
if (downloads[session_id][download_id]) { const session_downloads = downloads.find(session => session['session_id'] === session_id);
delete downloads[session_id][download_id]; if (session_downloads && session_downloads[download_id]) {
delete session_downloads[download_id];
success = true; success = true;
} else if (!downloads[session_id]) { } else if (!session_downloads) {
logger.error(`Session ${session_id} has no downloads.`) logger.error(`Session ${session_id} has no downloads.`)
} else if (!downloads[session_id][download_id]) { } else if (!session_downloads[download_id]) {
logger.error(`Download '${download_id}' for session '${session_id}' could not be found`); logger.error(`Download '${download_id}' for session '${session_id}' could not be found`);
} }
} else if (session_id) { } else if (session_id) {
// delete a session's downloads // delete a session's downloads
if (downloads[session_id]) { downloads = downloads.filter(session => session['session_id'] !== session_id);
delete downloads[session_id];
success = true;
} else {
logger.error(`Session ${session_id} has no downloads.`)
}
} }
updateDownloads(); updateDownloads();
res.send({success: success, downloads: downloads}); res.send({success: success, downloads: downloads});
@@ -2746,29 +2542,28 @@ app.post('/api/auth/changePassword', optionalJwt, async (req, res) => {
res.send({success: success}); res.send({success: success});
}); });
app.post('/api/auth/adminExists', async (req, res) => { app.post('/api/auth/adminExists', async (req, res) => {
let exists = auth_api.adminExists(); let exists = await auth_api.adminExists();
res.send({exists: exists}); res.send({exists: exists});
}); });
// user management // user management
app.post('/api/getUsers', optionalJwt, async (req, res) => { app.post('/api/getUsers', optionalJwt, async (req, res) => {
let users = users_db.get('users').value(); let users = await db_api.getRecords('users');
res.send({users: users}); res.send({users: users});
}); });
app.post('/api/getRoles', optionalJwt, async (req, res) => { app.post('/api/getRoles', optionalJwt, async (req, res) => {
let roles = users_db.get('roles').value(); let roles = await db_api.getRecords('roles');
res.send({roles: roles}); res.send({roles: roles});
}); });
app.post('/api/updateUser', optionalJwt, async (req, res) => { app.post('/api/updateUser', optionalJwt, async (req, res) => {
let change_obj = req.body.change_object; let change_obj = req.body.change_object;
try { try {
const user_db_obj = users_db.get('users').find({uid: change_obj.uid});
if (change_obj.name) { if (change_obj.name) {
user_db_obj.assign({name: change_obj.name}).write(); await db_api.updateRecord('users', {uid: change_obj.uid}, {name: change_obj.name});
} }
if (change_obj.role) { if (change_obj.role) {
user_db_obj.assign({role: change_obj.role}).write(); await db_api.updateRecord('users', {uid: change_obj.uid}, {role: change_obj.role});
} }
res.send({success: true}); res.send({success: true});
} catch (err) { } catch (err) {
@@ -2780,13 +2575,17 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
app.post('/api/deleteUser', optionalJwt, async (req, res) => { app.post('/api/deleteUser', optionalJwt, async (req, res) => {
let uid = req.body.uid; let uid = req.body.uid;
try { try {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid); const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = users_db.get('users').find({uid: uid}); const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj.value()) { if (user_db_obj) {
// user exists, let's delete // user exists, let's delete
await fs.remove(user_folder); await fs.remove(user_folder);
users_db.get('users').remove({uid: uid}).write(); await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
} }
res.send({success: true}); res.send({success: true});
} catch (err) { } catch (err) {
@@ -2805,7 +2604,7 @@ app.post('/api/changeUserPermissions', optionalJwt, async (req, res) => {
return; return;
} }
const success = auth_api.changeUserPermissions(user_uid, permission, new_value); const success = await auth_api.changeUserPermissions(user_uid, permission, new_value);
res.send({success: success}); res.send({success: success});
}); });
@@ -2820,7 +2619,7 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
return; return;
} }
const success = auth_api.changeRolePermissions(role, permission, new_value); const success = await auth_api.changeRolePermissions(role, permission, new_value);
res.send({success: success}); res.send({success: success});
}); });

4
backend/appdata/default.json
View File

@@ -54,6 +54,10 @@
"searchFilter": "(uid={{username}})" "searchFilter": "(uid={{username}})"
} }
}, },
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": { "Advanced": {
"default_downloader": "youtube-dl", "default_downloader": "youtube-dl",
"use_default_downloading_agent": true, "use_default_downloading_agent": true,

138
backend/authentication/auth.js
View File

@@ -13,16 +13,15 @@ var JwtStrategy = require('passport-jwt').Strategy,
// other required vars // other required vars
let logger = null; let logger = null;
let db = null; let db_api = null;
let users_db = null;
let SERVER_SECRET = null; let SERVER_SECRET = null;
let JWT_EXPIRATION = null; let JWT_EXPIRATION = null;
let opts = null; let opts = null;
let saltRounds = null; let saltRounds = null;
exports.initialize = function(input_db, input_users_db, input_logger) { exports.initialize = function(db_api, input_logger) {
setLogger(input_logger) setLogger(input_logger)
setDB(input_db, input_users_db); setDB(db_api);
/************************* /*************************
* Authentication module * Authentication module
@@ -32,21 +31,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration'); JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
SERVER_SECRET = null; SERVER_SECRET = null;
if (users_db.get('jwt_secret').value()) { if (db_api.users_db.get('jwt_secret').value()) {
SERVER_SECRET = users_db.get('jwt_secret').value(); SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
} else { } else {
SERVER_SECRET = uuid(); SERVER_SECRET = uuid();
users_db.set('jwt_secret', SERVER_SECRET).write(); db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
} }
opts = {} opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt'); opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET; opts.secretOrKey = SERVER_SECRET;
/*opts.issuer = 'example.com';
opts.audience = 'example.com';*/
exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) { exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = users_db.get('users').find({uid: jwt_payload.user}).value(); const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) { if (user) {
return done(null, user); return done(null, user);
} else { } else {
@@ -60,9 +57,8 @@ function setLogger(input_logger) {
logger = input_logger; logger = input_logger;
} }
function setDB(input_db, input_users_db) { function setDB(input_db_api) {
db = input_db; db_api = input_db_api;
users_db = input_users_db;
} }
exports.passport = require('passport'); exports.passport = require('passport');
@@ -78,7 +74,7 @@ exports.passport.deserializeUser(function(user, done) {
/*************************************** /***************************************
* Register user with hashed password * Register user with hashed password
**************************************/ **************************************/
exports.registerUser = function(req, res) { exports.registerUser = async function(req, res) {
var userid = req.body.userid; var userid = req.body.userid;
var username = req.body.username; var username = req.body.username;
var plaintextPassword = req.body.password; var plaintextPassword = req.body.password;
@@ -96,20 +92,20 @@ exports.registerUser = function(req, res) {
} }
bcrypt.hash(plaintextPassword, saltRounds) bcrypt.hash(plaintextPassword, saltRounds)
.then(function(hash) { .then(async function(hash) {
let new_user = generateUserObject(userid, username, hash); let new_user = generateUserObject(userid, username, hash);
// check if user exists // check if user exists
if (users_db.get('users').find({uid: userid}).value()) { if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken! // user id is taken!
logger.error('Registration failed: UID is already taken!'); logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!'); res.status(409).send('UID is already taken!');
} else if (users_db.get('users').find({name: username}).value()) { } else if (await db_api.getRecord('users', {name: username})) {
// user name is taken! // user name is taken!
logger.error('Registration failed: User name is already taken!'); logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!'); res.status(409).send('User name is already taken!');
} else { } else {
// add to db // add to db
users_db.get('users').push(new_user).write(); await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`); logger.verbose(`New user created: ${new_user.name}`);
res.send({ res.send({
user: new_user user: new_user
@@ -143,7 +139,7 @@ exports.registerUser = function(req, res) {
exports.login = async (username, password) => { exports.login = async (username, password) => {
const user = users_db.get('users').find({name: username}).value(); const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); false } if (!user) { logger.error(`User ${username} not found`); false }
if (user.auth_method && user.auth_method !== 'internal') { return false } if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false; return await bcrypt.compare(password, user.passhash) ? user : false;
@@ -164,17 +160,17 @@ var getLDAPConfiguration = function(req, callback) {
}; };
exports.passport.use(new LdapStrategy(getLDAPConfiguration, exports.passport.use(new LdapStrategy(getLDAPConfiguration,
function(user, done) { async function(user, done) {
// check if ldap auth is enabled // check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap'; const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false); if (!ldap_enabled) return done(null, false);
const user_uid = user.uid; const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value(); let db_user = await db_api.getRecord('users', {uid: user_uid});
if (!db_user) { if (!db_user) {
// generate DB user // generate DB user
let new_user = generateUserObject(user_uid, user_uid, null, 'ldap'); let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
users_db.get('users').push(new_user).write(); await db_api.insertRecordIntoTable('users', new_user);
db_user = new_user; db_user = new_user;
logger.verbose(`Generated new user ${user_uid} using LDAP`); logger.verbose(`Generated new user ${user_uid} using LDAP`);
} }
@@ -198,11 +194,11 @@ exports.generateJWT = function(req, res, next) {
next(); next();
} }
exports.returnAuthResponse = function(req, res) { exports.returnAuthResponse = async function(req, res) {
res.status(200).json({ res.status(200).json({
user: req.user, user: req.user,
token: req.token, token: req.token,
permissions: exports.userPermissions(req.user.uid), permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS'] available_permissions: consts['AVAILABLE_PERMISSIONS']
}); });
} }
@@ -215,7 +211,7 @@ exports.returnAuthResponse = function(req, res) {
* It also passes the user object to the next * It also passes the user object to the next
* middleware through res.locals * middleware through res.locals
**************************************/ **************************************/
exports.ensureAuthenticatedElseError = function(req, res, next) { exports.ensureAuthenticatedElseError = (req, res, next) => {
var token = getToken(req.query); var token = getToken(req.query);
if( token ) { if( token ) {
try { try {
@@ -233,10 +229,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
} }
// change password // change password
exports.changeUserPassword = async function(user_uid, new_pass) { exports.changeUserPassword = async (user_uid, new_pass) => {
try { try {
const hash = await bcrypt.hash(new_pass, saltRounds); const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write(); await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
return true; return true;
} catch (err) { } catch (err) {
return false; return false;
@@ -244,16 +240,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
} }
// change user permissions // change user permissions
exports.changeUserPermissions = function(user_uid, permission, new_value) { exports.changeUserPermissions = async (user_uid, permission, new_value) => {
try { try {
const user_db_obj = users_db.get('users').find({uid: user_uid}); await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
user_db_obj.get('permissions').pull(permission).write(); await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
user_db_obj.get('permission_overrides').pull(permission).write();
if (new_value === 'yes') { if (new_value === 'yes') {
user_db_obj.get('permissions').push(permission).write(); await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
user_db_obj.get('permission_overrides').push(permission).write(); await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} else if (new_value === 'no') { } else if (new_value === 'no') {
user_db_obj.get('permission_overrides').push(permission).write(); await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} }
return true; return true;
} catch (err) { } catch (err) {
@@ -263,12 +258,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
} }
// change role permissions // change role permissions
exports.changeRolePermissions = function(role, permission, new_value) { exports.changeRolePermissions = async (role, permission, new_value) => {
try { try {
const role_db_obj = users_db.get('roles').get(role); await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
role_db_obj.get('permissions').pull(permission).write();
if (new_value === 'yes') { if (new_value === 'yes') {
role_db_obj.get('permissions').push(permission).write(); await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
} }
return true; return true;
} catch (err) { } catch (err) {
@@ -277,19 +271,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
} }
} }
exports.adminExists = function() { exports.adminExists = async function() {
return !!users_db.get('users').find({uid: 'admin'}).value(); return !!(await db_api.getRecord('users', {uid: 'admin'}));
} }
// video stuff // video stuff
exports.getUserVideos = function(user_uid, type) { exports.getUserVideos = async function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value(); const files = await db_api.getRecords('files', {user_uid: user_uid});
return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files']; return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
} }
exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) { exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value(); let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info // prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null; if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -302,19 +296,17 @@ exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
return true; return true;
} }
exports.removePlaylist = function(user_uid, playlistID) { exports.removePlaylist = async function(user_uid, playlistID) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write(); await db_api.removeRecord('playlist', {playlistID: playlistID});
return true; return true;
} }
exports.getUserPlaylists = function(user_uid, user_files = null) { exports.getUserPlaylists = async function(user_uid, user_files = null) {
const user = users_db.get('users').find({uid: user_uid}).value(); return await db_api.getRecords('playlists', {user_uid: user_uid});
const playlists = JSON.parse(JSON.stringify(user['playlists']));
return playlists;
} }
exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) { exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value(); let playlist = await db_api.getRecord('playlists', {id: playlistID});
// prevent unauthorized users from accessing the file info // prevent unauthorized users from accessing the file info
if (requireSharing && !playlist['sharingEnabled']) playlist = null; if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -322,40 +314,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false)
return playlist; return playlist;
} }
exports.registerUserFile = function(user_uid, file_object) { exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
path: file_object['path']
}).write();
users_db.get('users').find({uid: user_uid}).get(`files`)
.push(file_object)
.write();
}
exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
let success = false; let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid}); is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
if (user_db_obj.value()) { success = true;
const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid});
if (file_db_obj.value()) {
success = true;
file_db_obj.assign({sharingEnabled: enabled}).write();
}
}
return success; return success;
} }
exports.userHasPermission = function(user_uid, permission) { exports.userHasPermission = async function(user_uid, permission) {
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role']; const role = user_obj['role'];
if (!role) { if (!role) {
// role doesn't exist // role doesn't exist
logger.error('Invalid role ' + role); logger.error('Invalid role ' + role);
return false; return false;
} }
const role_permissions = (users_db.get('roles').value())['permissions']; const role_permissions = (await db_api.getRecords('roles'))['permissions'];
const user_has_explicit_permission = user_obj['permissions'].includes(permission); const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission); const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -378,16 +353,17 @@ exports.userHasPermission = function(user_uid, permission) {
} }
} }
exports.userPermissions = function(user_uid) { exports.userPermissions = async function(user_uid) {
let user_permissions = []; let user_permissions = [];
const user_obj = users_db.get('users').find({uid: user_uid}).value(); const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role']; const role = user_obj['role'];
if (!role) { if (!role) {
// role doesn't exist // role doesn't exist
logger.error('Invalid role ' + role); logger.error('Invalid role ' + role);
return null; return null;
} }
const role_permissions = users_db.get('roles').get(role).get('permissions').value() const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) { for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i]; let permission = consts['AVAILABLE_PERMISSIONS'][i];

17
backend/categories.js
View File

@@ -39,10 +39,9 @@ async function categorize(file_jsons) {
if (!Array.isArray(file_jsons)) file_jsons = [file_jsons]; if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
let selected_category = null; let selected_category = null;
const categories = getCategories(); const categories = await getCategories();
if (!categories) { if (!categories) {
logger.warn('Categories could not be found. Initializing categories...'); logger.warn('Categories could not be found.');
db.assign({categories: []}).write();
return null; return null;
} }
@@ -64,14 +63,14 @@ async function categorize(file_jsons) {
return selected_category; return selected_category;
} }
function getCategories() { async function getCategories() {
const categories = db.get('categories').value(); const categories = await db_api.getRecords('categories');
return categories ? categories : null; return categories ? categories : null;
} }
function getCategoriesAsPlaylists(files = null) { async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = []; const categories_as_playlists = [];
const available_categories = getCategories(); const available_categories = await getCategories();
if (available_categories && files) { if (available_categories && files) {
for (category of available_categories) { for (category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files); const files_that_match = utils.addUIDsToCategory(category, files);
@@ -97,10 +96,10 @@ function applyCategoryRules(file_json, rules, category_name) {
switch (rule['comparator']) { switch (rule['comparator']) {
case 'includes': case 'includes':
rule_applies = file_json[rule['property']].includes(rule['value']); rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
break; break;
case 'not_includes': case 'not_includes':
rule_applies = !(file_json[rule['property']].includes(rule['value'])); rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
break; break;
case 'equals': case 'equals':
rule_applies = file_json[rule['property']] === rule['value']; rule_applies = file_json[rule['property']] === rule['value'];

4
backend/config.js
View File

@@ -231,6 +231,10 @@ DEFAULT_CONFIG = {
"searchFilter": "(uid={{username}})" "searchFilter": "(uid={{username}})"
} }
}, },
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": { "Advanced": {
"default_downloader": "youtube-dl", "default_downloader": "youtube-dl",
"use_default_downloading_agent": true, "use_default_downloading_agent": true,

10
backend/consts.js
View File

@@ -153,6 +153,16 @@ let CONFIG_ITEMS = {
'path': 'YoutubeDLMaterial.Users.ldap_config' 'path': 'YoutubeDLMaterial.Users.ldap_config'
}, },
// Database
'ytdl_use_local_db': {
'key': 'ytdl_use_local_db',
'path': 'YoutubeDLMaterial.Database.use_local_db'
},
'ytdl_mongodb_connection_string': {
'key': 'ytdl_mongodb_connection_string',
'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
},
// Advanced // Advanced
'ytdl_default_downloader': { 'ytdl_default_downloader': {
'key': 'ytdl_default_downloader', 'key': 'ytdl_default_downloader',

764
backend/db.js
View File

@@ -3,22 +3,102 @@ var path = require('path')
var utils = require('./utils') var utils = require('./utils')
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
const config_api = require('./config'); const config_api = require('./config');
const { MongoClient } = require("mongodb");
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync');
const local_adapter = new FileSync('./appdata/local_db.json');
const local_db = low(local_adapter);
var logger = null; var logger = null;
var db = null; var db = null;
var users_db = null; var users_db = null;
function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db } var database = null;
function setLogger(input_logger) { logger = input_logger; }
const tables = ['files', 'playlists', 'categories', 'subscriptions', 'downloads', 'users', 'roles', 'test'];
const local_db_defaults = {}
tables.forEach(table => {local_db_defaults[table] = []});
local_db.defaults(local_db_defaults).write();
let using_local_db = config_api.getConfigItem('ytdl_use_local_db');
function setDB(input_db, input_users_db) {
db = input_db; users_db = input_users_db;
exports.db = input_db;
exports.users_db = input_users_db
}
function setLogger(input_logger) {
logger = input_logger;
}
exports.initialize = (input_db, input_users_db, input_logger) => { exports.initialize = (input_db, input_users_db, input_logger) => {
setDB(input_db, input_users_db); setDB(input_db, input_users_db);
setLogger(input_logger); setLogger(input_logger);
} }
exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null) => { exports.connectToDB = async (retries = 5, no_fallback = false) => {
if (using_local_db) return;
const success = await exports._connectToDB();
if (success) return true;
logger.warn(`MongoDB connection failed! Retrying ${retries} times...`);
const retry_delay_ms = 2000;
for (let i = 0; i < retries; i++) {
const retry_succeeded = await exports._connectToDB();
if (retry_succeeded) {
logger.info(`Successfully connected to DB after ${i+1} attempt(s)`);
return true;
}
if (i !== retries - 1) {
logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`);
await utils.wait(retry_delay_ms);
} else {
logger.warn(`Retry ${i+1} failed.`);
}
}
if (no_fallback) {
logger.error('Failed to connect to MongoDB. Verify your connection string is valid.');
return;
}
using_local_db = true;
config_api.setConfigItem('ytdl_use_local_db', true);
logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.');
return true;
}
exports._connectToDB = async () => {
const uri = config_api.getConfigItem('ytdl_mongodb_connection_string'); // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb";
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
try {
await client.connect();
database = client.db('ytdl_material');
const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name);
const missing_tables = tables.filter(table => !(existing_collections.includes(table)));
missing_tables.forEach(async table => {
await database.createCollection(table);
})
return true;
} catch(err) {
logger.error(err);
return false;
} finally {
// Ensures that the client will close when you finish/error
// await client.close();
}
}
exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => {
let db_path = null; let db_path = null;
const file_id = utils.removeFileExtension(file_path); const file_id = utils.removeFileExtension(file_path);
const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
if (!file_object) { if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
return false; return false;
@@ -37,23 +117,9 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
} }
if (!sub) { if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
if (multiUserMode) {
const user_uid = multiUserMode.user;
db_path = users_db.get('users').find({uid: user_uid}).get(`files`);
} else {
db_path = db.get(`files`);
}
} else {
if (multiUserMode) {
const user_uid = multiUserMode.user;
db_path = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
} else {
db_path = db.get('subscriptions').find({id: sub.id}).get('videos');
}
}
const file_obj = registerFileDBManual(db_path, file_object); const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed // remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) { if (!config_api.getConfigItem('ytdl_include_metadata')) {
@@ -63,18 +129,48 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
return file_obj; return file_obj;
} }
function registerFileDBManual(db_path, file_object) { exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject2(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms2(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile2(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info // add additional info
file_object['uid'] = uuid(); file_object['uid'] = uuid();
file_object['registered'] = Date.now(); file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']); path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object); file_object['path'] = path.format(path_object);
// remove duplicate(s) exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
db_path.remove({path: file_object['path']}).write();
// add new file to db
db_path.push(file_object).write();
return file_object; return file_object;
} }
@@ -107,11 +203,38 @@ function generateFileObject(id, type, customPath = null, sub = null) {
return file_obj; return file_obj;
} }
function generateFileObject2(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
var stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function getAppendedBasePathSub(sub, base_path) { function getAppendedBasePathSub(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
} }
exports.getFileDirectoriesAndDBs = () => { exports.getFileDirectoriesAndDBs = async () => {
let dirs_to_check = []; let dirs_to_check = [];
let subscriptions_to_check = []; let subscriptions_to_check = [];
const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode
@@ -119,48 +242,45 @@ exports.getFileDirectoriesAndDBs = () => {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions'); const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions');
if (multi_user_mode) { if (multi_user_mode) {
let users = users_db.get('users').value(); const users = await exports.getRecords('users');
for (let i = 0; i < users.length; i++) { for (let i = 0; i < users.length; i++) {
const user = users[i]; const user = users[i];
if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']);
// add user's audio dir to check list // add user's audio dir to check list
dirs_to_check.push({ dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'audio'), basePath: path.join(usersFileFolder, user.uid, 'audio'),
dbPath: users_db.get('users').find({uid: user.uid}).get('files'), user_uid: user.uid,
type: 'audio' type: 'audio'
}); });
// add user's video dir to check list // add user's video dir to check list
dirs_to_check.push({ dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'video'), basePath: path.join(usersFileFolder, user.uid, 'video'),
dbPath: users_db.get('users').find({uid: user.uid}).get('files'),
type: 'video' type: 'video'
}); });
} }
} else { } else {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const subscriptions = db.get('subscriptions').value();
if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
// add audio dir to check list // add audio dir to check list
dirs_to_check.push({ dirs_to_check.push({
basePath: audioFolderPath, basePath: audioFolderPath,
dbPath: db.get('files'),
type: 'audio' type: 'audio'
}); });
// add video dir to check list // add video dir to check list
dirs_to_check.push({ dirs_to_check.push({
basePath: videoFolderPath, basePath: videoFolderPath,
dbPath: db.get('files'),
type: 'video' type: 'video'
}); });
} }
if (subscriptions_enabled) {
const subscriptions = await exports.getRecords('subscriptions');
subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
}
// add subscriptions to check list // add subscriptions to check list
for (let i = 0; i < subscriptions_to_check.length; i++) { for (let i = 0; i < subscriptions_to_check.length; i++) {
let subscription_to_check = subscriptions_to_check[i]; let subscription_to_check = subscriptions_to_check[i];
@@ -169,11 +289,11 @@ exports.getFileDirectoriesAndDBs = () => {
continue; continue;
} }
dirs_to_check.push({ dirs_to_check.push({
basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name), : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos') user_uid: subscription_to_check.user_uid,
: db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'), type: subscription_to_check.type,
type: subscription_to_check.type sub_id: subscription_to_check['id']
}); });
} }
@@ -181,22 +301,25 @@ exports.getFileDirectoriesAndDBs = () => {
} }
exports.importUnregisteredFiles = async () => { exports.importUnregisteredFiles = async () => {
const dirs_to_check = exports.getFileDirectoriesAndDBs(); const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db // run through check list and check each file to see if it's missing from the db
for (const dir_to_check of dirs_to_check) { for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path // recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => { for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it // check if file exists in db, if not add it
const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value()) const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id}))
if (!file_is_registered) { if (!file_is_registered) {
// add additional info // add additional info
registerFileDBManual(dir_to_check.dbPath, file); await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
logger.verbose(`Added discovered file to the database: ${file.id}`); logger.verbose(`Added discovered file to the database: ${file.id}`);
} }
}); }
} }
} }
@@ -204,26 +327,45 @@ exports.importUnregisteredFiles = async () => {
exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => { exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
const preimported_file_paths = []; const preimported_file_paths = [];
let dbPath = null;
if (sub.user_uid)
dbPath = users_db.get('users').find({uid: sub.user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
else
dbPath = db.get('subscriptions').find({id: sub.id}).get('videos');
const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type); const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
files.forEach(file => { for (let i = 0; i < files.length; i++) {
const file = files[i];
// check if file exists in db, if not add it // check if file exists in db, if not add it
const file_is_registered = !!(dbPath.find({id: file.id}).value()) const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
if (!file_is_registered) { if (!file_is_registered) {
// add additional info // add additional info
registerFileDBManual(dbPath, file); await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
preimported_file_paths.push(file['path']); preimported_file_paths.push(file['path']);
logger.verbose(`Preemptively added subscription file to the database: ${file.id}`); logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
} }
}); }
return preimported_file_paths; return preimported_file_paths;
} }
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await exports.bulkUpdateRecords('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => { exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => {
let new_playlist = { let new_playlist = {
name: playlist_name, name: playlist_name,
@@ -237,27 +379,18 @@ exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_u
const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid); const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid);
new_playlist.duration = duration; new_playlist.duration = duration;
if (user_uid) { new_playlist.user_uid = user_uid ? user_uid : undefined;
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
} else { await exports.insertRecordIntoTable('playlists', new_playlist);
db.get(`playlists`)
.push(new_playlist)
.write();
}
return new_playlist; return new_playlist;
} }
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => { exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = null let playlist = await exports.getRecord('playlists', {id: playlist_id});
if (user_uid) {
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlist_id}).value();
} else {
playlist = db.get(`playlists`).find({id: playlist_id}).value();
}
if (!playlist) { if (!playlist) {
playlist = db.get('categories').find({uid: playlist_id}).value(); playlist = await exports.getRecord('categories', {uid: playlist_id});
if (playlist) { if (playlist) {
// category found // category found
const files = await exports.getFiles(user_uid); const files = await exports.getFiles(user_uid);
@@ -271,7 +404,7 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`); logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) { for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i]; const fileName = playlist['fileNames'][i];
const uid = exports.getVideoUIDByID(fileName, user_uid); const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid); if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
} }
@@ -290,14 +423,21 @@ exports.updatePlaylist = async (playlist, user_uid = null) => {
const duration = await exports.calculatePlaylistDuration(playlist, user_uid); const duration = await exports.calculatePlaylistDuration(playlist, user_uid);
playlist.duration = duration; playlist.duration = duration;
let db_loc = null; return await exports.updateRecord('playlists', {id: playlistID}, playlist);
if (user_uid) { }
db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID});
} else { exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
db_loc = db.get(`playlists`).find({id: playlistID}); let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
} }
db_loc.assign(playlist).write();
return true; if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
} }
exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => { exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => {
@@ -381,8 +521,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
if (jsonExists) await fs.unlink(jsonPath); if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath); if (thumbnailExists) await fs.unlink(thumbnailPath);
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; await exports.removeRecord('files', {uid: uid});
base_db_path.get('files').remove({uid: uid}).write();
if (fileExists) { if (fileExists) {
await fs.unlink(file_obj.path); await fs.unlink(file_obj.path);
@@ -398,29 +537,462 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
} }
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that // Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = (file_id, uuid = null) => { exports.getVideoUIDByID = async (file_id, uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; const file_obj = await exports.getRecord('files', {id: file_id});
const file_obj = base_db_path.get('files').find({id: file_id}).value();
return file_obj ? file_obj['uid'] : null; return file_obj ? file_obj['uid'] : null;
} }
exports.getVideo = async (file_uid, uuid = null, sub_id = null) => { exports.getVideo = async (file_uid, uuid = null, sub_id = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; return await exports.getRecord('files', {uid: file_uid});
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
return sub_db_path.find({uid: file_uid}).value();
} }
exports.getFiles = async (uuid = null) => { exports.getFiles = async (uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; return await exports.getRecords('files', {user_uid: uuid});
return base_db_path.get('files').value();
} }
exports.setVideoProperty = async (file_uid, assignment_obj, uuid, sub_id) => { exports.setVideoProperty = async (file_uid, assignment_obj) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db; // TODO: check if video exists, throw error if not
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files'); await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
const file_db_path = sub_db_path.find({uid: file_uid}); }
if (!(file_db_path.value())) {
logger.error(`Failed to find file with uid ${file_uid}`); // DB to JSON
}
sub_db_path.find({uid: file_uid}).assign(assignment_obj).write(); exports.exportDBToJSON = async (tables) => {
const users_db_json = await createUsersJSONs(tables.files, tables.playlists, tables.subscriptions, tables.categories, tables.users);
const db_json = await createNonUserJSON(tables.files, tables.playlists, tables.subscriptions, tables.categories);
return {users_db_json: users_db_json, db_json: db_json};
}
const createUsersJSONs = async (files, playlists, subscriptions, categories, users) => {
// we need to already have a list of user objects to gather the records into
for (let user of users) {
const files_of_user = files.filter(file => file.user_uid === user.uid && !file.sub_id);
const playlists_of_user = playlists.filter(playlist => playlist.user_uid === user.uid);
const subs_of_user = subscriptions.filter(sub => sub.user_uid === user.uid);
const categories_of_user = categories ? categories.filter(category => category && category.user_uid === user.uid) : [];
user['files'] = files_of_user;
user['playlists'] = playlists_of_user;
user['subscriptions'] = subs_of_user;
user['categories'] = categories_of_user;
for (let subscription of subscriptions) {
subscription['videos'] = files.filter(file => file.user_uid === user.uid && file.sub_id === sub.id);
}
}
}
const createNonUserJSON = async (files, playlists, subscriptions, categories) => {
const non_user_json = {
files: files.filter(file => !file.user_uid && !file.sub_id),
playlists: playlists.filter(playlist => !playlist.user_uid),
subscriptions: subscriptions.filter(sub => !sub.user_uid),
categories: categories ? categories.filter(category => category && !category.user_uid) : []
}
for (let subscription of non_user_json['subscriptions']) {
subscription['videos'] = files.filter(file => !file.user_uid && file.sub_id === subscription.id);
}
return non_user_json;
}
// Basic DB functions
// Create
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override
if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write();
return true;
}
if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
const output = await database.collection(table).insertOne(doc);
logger.debug(`Inserted doc into ${table}`);
return !!(output['result']['ok']);
}
exports.insertRecordsIntoTable = async (table, docs) => {
// local db override
if (using_local_db) {
local_db.get(table).push(...docs).write();
return true;
}
const output = await database.collection(table).insertMany(docs);
logger.debug(`Inserted ${output.insertedCount} docs into ${table}`);
return !!(output['result']['ok']);
}
exports.bulkInsertRecordsIntoTable = async (table, docs) => {
// local db override
if (using_local_db) {
return await exports.insertRecordsIntoTable(table, docs);
}
// not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed
const table_collection = database.collection(table);
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
for (let i = 0; i < docs.length; i++) {
bulk.insert(docs[i]);
}
const output = await bulk.execute();
return !!(output['result']['ok']);
}
// Read
exports.getRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
}
return await database.collection(table).findOne(filter_obj);
}
exports.getRecords = async (table, filter_obj = null) => {
// local db override
if (using_local_db) {
return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
}
return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray();
}
// Update
exports.updateRecord = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
return true;
}
// sometimes _id will be in the update obj, this breaks mongodb
if (update_obj['_id']) delete update_obj['_id'];
const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj});
return !!(output['result']['ok']);
}
exports.updateRecords = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
return true;
}
const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj});
return !!(output['result']['ok']);
}
exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
// local db override
if (using_local_db) {
local_db.get(table).each((record) => {
const item_id_to_update = record[key_label];
if (!update_obj[item_id_to_update]) return;
const props_to_update = Object.keys(update_obj[item_id_to_update]);
for (let i = 0; i < props_to_update.length; i++) {
const prop_to_update = props_to_update[i];
const prop_value = update_obj[item_id_to_update][prop_to_update];
record[prop_to_update] = prop_value;
}
}).write();
return true;
}
const table_collection = database.collection(table);
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
const item_ids_to_update = Object.keys(update_obj);
for (let i = 0; i < item_ids_to_update.length; i++) {
const item_id_to_update = item_ids_to_update[i];
bulk.find({[key_label]: item_id_to_update }).updateOne({
"$set": update_obj[item_id_to_update]
});
}
const output = await bulk.execute();
return !!(output['result']['ok']);
}
exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}});
return !!(output['result']['ok']);
}
exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}});
return !!(output['result']['ok']);
}
// Delete
exports.removeRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
return true;
}
const output = await database.collection(table).deleteOne(filter_obj);
return !!(output['result']['ok']);
}
exports.removeAllRecords = async (table = null) => {
// local db override
if (using_local_db) {
const tables_to_remove = table ? [table] : tables;
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
local_db.assign({[table_to_remove]: []}).write();
logger.debug(`Removed all records from ${table_to_remove}`);
}
return true;
}
let success = true;
const tables_to_remove = table ? [table] : tables;
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
const output = await database.collection(table_to_remove).deleteMany({});
logger.debug(`Removed all records from ${table_to_remove}`);
success &= !!(output['result']['ok']);
}
return success;
}
// Stats
exports.getDBStats = async () => {
const stats_by_table = {};
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
if (table === 'test') continue;
stats_by_table[table] = await getDBTableStats(table);
}
return {stats_by_table: stats_by_table, using_local_db: using_local_db};
}
const getDBTableStats = async (table) => {
const table_stats = {};
// local db override
if (using_local_db) {
table_stats['records_count'] = local_db.get(table).value().length;
} else {
const stats = await database.collection(table).stats();
table_stats['records_count'] = stats.count;
}
return table_stats;
}
// JSON to DB
exports.generateJSONTables = async (db_json, users_json) => {
// create records
let files = db_json['files'] || [];
let playlists = db_json['playlists'] || [];
let categories = db_json['categories'] || [];
let subscriptions = db_json['subscriptions'] || [];
const users = users_json['users'];
for (let i = 0; i < users.length; i++) {
const user = users[i];
if (user['files']) {
user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] }));
files = files.concat(user['files']);
}
if (user['playlists']) {
user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] }));
playlists = playlists.concat(user['playlists']);
}
if (user['categories']) {
user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] }));
categories = categories.concat(user['categories']);
}
if (user['subscriptions']) {
user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] }));
subscriptions = subscriptions.concat(user['subscriptions']);
}
}
const tables_obj = {};
// TODO: use create*Records funcs to strip unnecessary properties
tables_obj.files = createFilesRecords(files, subscriptions);
tables_obj.playlists = playlists;
tables_obj.categories = categories;
tables_obj.subscriptions = createSubscriptionsRecords(subscriptions);
tables_obj.users = createUsersRecords(users);
tables_obj.roles = createRolesRecords(users_json['roles']);
tables_obj.downloads = createDownloadsRecords(db_json['downloads'])
return tables_obj;
}
exports.importJSONToDB = async (db_json, users_json) => {
// TODO: backup db
// TODO: delete current records
const tables_obj = await exports.generateJSONTables(db_json, users_json);
const table_keys = Object.keys(tables_obj);
let success = true;
for (let i = 0; i < table_keys.length; i++) {
const table_key = table_keys[i];
success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key]);
}
return success;
}
const createFilesRecords = (files, subscriptions) => {
for (let i = 0; i < subscriptions.length; i++) {
const subscription = subscriptions[i];
subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
files = files.concat(subscriptions[i]['videos']);
console.log(files.length);
}
return files;
}
const createPlaylistsRecords = async (playlists) => {
}
const createCategoriesRecords = async (categories) => {
}
const createSubscriptionsRecords = (subscriptions) => {
for (let i = 0; i < subscriptions.length; i++) {
delete subscriptions[i]['videos'];
}
return subscriptions;
}
const createUsersRecords = (users) => {
users.forEach(user => {
delete user['files'];
delete user['playlists'];
delete user['subscriptions'];
});
return users;
}
const createRolesRecords = (roles) => {
const new_roles = [];
Object.keys(roles).forEach(role_key => {
new_roles.push({
key: role_key,
...roles[role_key]
});
});
return new_roles;
}
const createDownloadsRecords = (downloads) => {
const new_downloads = [];
Object.keys(downloads).forEach(session_key => {
new_downloads.push({
key: session_key,
...downloads[session_key]
});
});
return new_downloads;
}
exports.transferDB = async (local_to_remote) => {
const table_to_records = {};
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
table_to_records[table] = await exports.getRecords(table);
}
using_local_db = !local_to_remote;
if (local_to_remote) {
// backup local DB
logger.debug('Backup up Local DB...');
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
const db_connected = await exports.connectToDB(5, true);
if (!db_connected) {
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
return false;
}
}
success = true;
logger.debug('Clearing new database before transfer...');
await exports.removeAllRecords();
logger.debug('Database cleared! Beginning transfer.');
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
}
config_api.setConfigItem('ytdl_use_local_db', using_local_db);
return success;
}
/*
This function is necessary to emulate mongodb's ability to search for null or missing values.
A filter of null or undefined for a property will find docs that have that property missing, or have it
null or undefined. We want that same functionality for the local DB as well
*/
const applyFilterLocalDB = (db_path, filter_obj, operation) => {
const filter_props = Object.keys(filter_obj);
const return_val = db_path[operation](record => {
if (!filter_props) return true;
let filtered = true;
for (let i = 0; i < filter_props.length; i++) {
const filter_prop = filter_props[i];
const filter_prop_value = filter_obj[filter_prop];
if (filter_prop_value === undefined || filter_prop_value === null) {
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
} else {
filtered &= record[filter_prop] === filter_prop_value;
}
}
return filtered;
});
return return_val;
} }

77
backend/package-lock.json generated
View File

@@ -441,6 +441,11 @@
"resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz",
"integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw=="
}, },
"bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg=="
},
"buffer": { "buffer": {
"version": "5.5.0", "version": "5.5.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
@@ -923,6 +928,11 @@
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
}, },
"denque": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
},
"depd": { "depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -2042,6 +2052,12 @@
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
}, },
"memory-pager": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
"optional": true
},
"merge-descriptors": { "merge-descriptors": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
@@ -2196,6 +2212,44 @@
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
}, },
"mongodb": {
"version": "3.6.9",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.9.tgz",
"integrity": "sha512-1nSCKgSunzn/CXwgOWgbPHUWOO5OfERcuOWISmqd610jn0s8BU9K4879iJVabqgpPPbA6hO7rG48eq+fGED3Mg==",
"requires": {
"bl": "^2.2.1",
"bson": "^1.1.4",
"denque": "^1.4.1",
"optional-require": "^1.0.3",
"safe-buffer": "^5.1.2",
"saslprep": "^1.0.0"
},
"dependencies": {
"bl": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
}
}
},
"ms": { "ms": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -2423,6 +2477,11 @@
"mimic-fn": "^2.1.0" "mimic-fn": "^2.1.0"
} }
}, },
"optional-require": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.3.tgz",
"integrity": "sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA=="
},
"p-cancelable": { "p-cancelable": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz",
@@ -2820,6 +2879,15 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
}, },
"saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
"optional": true,
"requires": {
"sparse-bitfield": "^3.0.3"
}
},
"semver": { "semver": {
"version": "5.7.1", "version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
@@ -2930,6 +2998,15 @@
"is-arrayish": "^0.3.1" "is-arrayish": "^0.3.1"
} }
}, },
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
"optional": true,
"requires": {
"memory-pager": "^1.0.2"
}
},
"sshpk": { "sshpk": {
"version": "1.16.1", "version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",

4
backend/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"test": "echo \"Error: no test specified\" && exit 1", "test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon app.js" "start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
}, },
"nodemonConfig": { "nodemonConfig": {
"ignore": [ "ignore": [
@@ -46,6 +47,7 @@
"merge-files": "^0.1.2", "merge-files": "^0.1.2",
"mocha": "^8.4.0", "mocha": "^8.4.0",
"moment": "^2.29.1", "moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2", "multer": "^1.4.2",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"node-id3": "^0.1.14", "node-id3": "^0.1.14",

164
backend/subscriptions.js
View File

@@ -14,13 +14,13 @@ const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null; var logger = null;
var db = null; var db = null;
var users_db = null; var users_db = null;
var db_api = null; let db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } function setDB(input_db_api) { db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; } function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger, input_db_api) { function initialize(input_db_api, input_logger) {
setDB(input_db, input_users_db, input_db_api); setDB(input_db_api);
setLogger(input_logger); setLogger(input_logger);
} }
@@ -34,12 +34,7 @@ async function subscribe(sub, user_uid = null) {
sub.isPlaylist = sub.url.includes('playlist'); sub.isPlaylist = sub.url.includes('playlist');
sub.videos = []; sub.videos = [];
let url_exists = false; let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
if (user_uid)
url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
else
url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
if (!sub.name && url_exists) { if (!sub.name && url_exists) {
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`); logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -48,19 +43,12 @@ async function subscribe(sub, user_uid = null) {
return; return;
} }
// add sub to db sub['user_uid'] = user_uid ? user_uid : undefined;
let sub_db = null; await db_api.insertRecordIntoTable('subscriptions', sub);
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
let success = await getSubscriptionInfo(sub, user_uid); let success = await getSubscriptionInfo(sub, user_uid);
if (success) { if (success) {
sub = sub_db.value();
getVideosForSub(sub, user_uid); getVideosForSub(sub, user_uid);
} else { } else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
@@ -91,8 +79,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
} }
} }
return new Promise(resolve => { return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, function(err, output) { youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (debugMode) { if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id); logger.info('Subscribe: got info for subscription ' + sub.id);
} }
@@ -122,10 +110,7 @@ async function getSubscriptionInfo(sub, user_uid = null) {
} }
// if it's now valid, update // if it's now valid, update
if (sub.name) { if (sub.name) {
if (user_uid) await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
} }
} }
@@ -141,10 +126,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
// updates subscription // updates subscription
sub.archive = archive_dir; sub.archive = archive_dir;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
else
db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
} }
// TODO: get even more info // TODO: get even more info
@@ -166,10 +149,8 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
let result_obj = { success: false, error: '' }; let result_obj = { success: false, error: '' };
let id = sub.id; let id = sub.id;
if (user_uid) await db_api.removeRecord('subscriptions', {id: id});
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write(); await db_api.removeAllRecords('files', {sub_id: id});
else
db.get('subscriptions').remove({id: id}).write();
// failed subs have no name, on unsubscribe they shouldn't error // failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) { if (!sub.name) {
@@ -191,20 +172,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
} }
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
// TODO: combine this with deletefile
let basePath = null; let basePath = null;
let sub_db = null; basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
if (user_uid) { : config_api.getConfigItem('ytdl_subscriptions_base_path');
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
sub_db = db.get('subscriptions').find({id: sub.id});
}
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath); const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file; const name = file;
let retrievedID = null; let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
await db_api.removeRecord('files', {uid: file_uid});
let filePath = appendedBasePath; let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
@@ -255,14 +232,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
} }
async function getVideosForSub(sub, user_uid = null) { async function getVideosForSub(sub, user_uid = null) {
// get sub_db const latest_sub_obj = await getSubscription(sub.id);
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
const latest_sub_obj = sub_db.value();
if (!latest_sub_obj || latest_sub_obj['downloading']) { if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false; return false;
} }
@@ -292,12 +262,12 @@ async function getVideosForSub(sub, user_uid = null) {
// get videos // get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name); logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => { return new Promise(async resolve => {
const preimported_file_paths = []; const preimported_file_paths = [];
const PREIMPORT_INTERVAL = 5000; const PREIMPORT_INTERVAL = 5000;
const preregister_check = setInterval(() => { const preregister_check = setInterval(async () => {
if (sub.streamingOnly) return; if (sub.streamingOnly) return;
db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath); await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
}, PREIMPORT_INTERVAL); }, PREIMPORT_INTERVAL);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup // cleanup
@@ -313,7 +283,7 @@ async function getVideosForSub(sub, user_uid = null) {
const outputs = err.stdout.split(/\r\n|\r|\n/); const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) { for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]); const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) await handleOutputJSON(sub, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) { if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors // we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) { if (sub.archive) {
@@ -347,7 +317,7 @@ async function getVideosForSub(sub, user_uid = null) {
} }
const reset_videos = i === 0; const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, preimported_file_paths, reset_videos); await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
} }
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
@@ -444,8 +414,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
return downloadConfig; return downloadConfig;
} }
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) { // TODO: remove streaming only mode
if (false && sub.streamingOnly) {
if (reset_videos) { if (reset_videos) {
sub_db.assign({videos: []}).write(); sub_db.assign({videos: []}).write();
} }
@@ -459,12 +430,15 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
path_object = path.parse(output_json['_filename']); path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object); const path_string = path.format(path_object);
if (sub_db.get('videos').find({path: path_string}).value()) { const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id});
if (file_exists) {
// TODO: fix issue where files of different paths due to custom path get downloaded multiple times
// file already exists in DB, return early to avoid reseting the download date // file already exists in DB, return early to avoid reseting the download date
return; return;
} }
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);
const url = output_json['webpage_url']; const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) { && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
@@ -477,73 +451,41 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
} }
} }
function getSubscriptions(user_uid = null) { async function getSubscriptions(user_uid = null) {
if (user_uid) return await db_api.getRecords('subscriptions', {user_uid: user_uid});
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
else
return db.get('subscriptions').value();
} }
function getAllSubscriptions() { async function getAllSubscriptions() {
let subscriptions = null; const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (multiUserMode) { return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
subscriptions = [];
let users = users_db.get('users').value();
for (let i = 0; i < users.length; i++) {
if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']);
}
} else {
subscriptions = getSubscriptions();
}
return subscriptions;
} }
function getSubscription(subID, user_uid = null) { async function getSubscription(subID) {
if (user_uid) return await db_api.getRecord('subscriptions', {id: subID});
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return db.get('subscriptions').find({id: subID}).value();
} }
function getSubscriptionByName(subName, user_uid = null) { async function getSubscriptionByName(subName, user_uid = null) {
if (user_uid) return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value();
else
return db.get('subscriptions').find({name: subName}).value();
} }
function updateSubscription(sub, user_uid = null) { async function updateSubscription(sub, user_uid = null) {
if (user_uid) { await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(sub).write();
}
return true; return true;
} }
function updateSubscriptionPropertyMultiple(subs, assignment_obj) { async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(sub => { subs.forEach(async sub => {
updateSubscriptionProperty(sub, assignment_obj, sub.user_uid); await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
}); });
} }
function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) { async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
if (user_uid) { // TODO: combine with updateSubscription
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write(); await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
} else {
db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
}
return true; return true;
} }
function subExists(subID, user_uid = null) {
if (user_uid)
return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return !!db.get('subscriptions').find({id: subID}).value();
}
async function setFreshUploads(sub, user_uid) { async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => { sub.videos.forEach(async video => {
@@ -559,7 +501,7 @@ async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => { sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) { if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
checkVideoIfBetterExists(video, sub, user_uid) await checkVideoIfBetterExists(video, sub, user_uid)
} }
}); });
} }
@@ -569,7 +511,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path); const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists // simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => { youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
if (err) { if (err) {
// video is not available anymore for whatever reason // video is not available anymore for whatever reason
} else if (output) { } else if (output) {

165
backend/test/tests.js
View File

@@ -29,7 +29,7 @@ const logger = winston.createLogger({
// //
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }), new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'}) new winston.transports.Console({level: 'debug', name: 'console'})
] ]
}); });
@@ -38,19 +38,167 @@ var db_api = require('../db');
const utils = require('../utils'); const utils = require('../utils');
const subscriptions_api = require('../subscriptions'); const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra'); const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db, logger); db_api.initialize(db, users_db, logger);
auth_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api);
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(true);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Transfer to local', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(false);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
});
describe('Export', function() {
});
describe('Import and Export', async function() {
it('Existing data', async function() {
const users_db_json = users_db.value();
const db_json = db.value();
const users_db_json_stringified = JSON.stringify(users_db_json);
const db_json_stringified = JSON.stringify(db_json);
const tables_obj = await db_api.importJSONtoDB(users_db_json, db_json);
const db_jsons = await db_api.exportDBToJSON(tables_obj);
const users_db_json_returned_stringified = db_jsons['users_db_json'];
const db_json_returned_stringified = db_jsons['db_json'];
assert(users_db_json_returned_stringified.length === users_db_json_stringified.length);
assert(db_json_returned_stringified.length === db_json_stringified.length);
});
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
});
});
describe('Multi User', async function() { describe('Multi User', async function() {
let user = null; let user = null;
const user_to_test = 'admin'; const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x'; const playlist_to_test = 'ysabVZz4x';
before(async function() { beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass'); user = await auth_api.login('admin', 'pass');
console.log('hi')
}); });
describe('Authentication', function() { describe('Authentication', function() {
it('login', async function() { it('login', async function() {
@@ -93,11 +241,12 @@ describe('Multi User', async function() {
}); });
it('Subscription zip generator', async function() { it('Subscription zip generator', async function() {
const sub = subscriptions_api.getSubscription(sub_to_test, user_to_test); const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
assert(sub); assert(sub);
const sub_files_to_download = []; const sub_files_to_download = [];
for (let i = 0; i < sub['videos'].length; i++) { for (let i = 0; i < sub_videos.length; i++) {
const sub_file = sub['videos'][i]; const sub_file = sub_videos[i];
sub_files_to_download.push(sub_file); sub_files_to_download.push(sub_file);
} }
const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download); const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);

70
backend/utils.js
View File

@@ -122,6 +122,21 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj; return obj;
} }
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
}
else obj = 0;
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) { function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
} }
@@ -143,6 +158,23 @@ function getDownloadedThumbnail(name, type, customPath = null) {
return null; return null;
} }
function getDownloadedThumbnail2(file_path, type) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
let pngPath = file_path_no_extension + '.png';
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getExpectedFileSize(input_info_jsons) { function getExpectedFileSize(input_info_jsons) {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons]; const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
@@ -190,6 +222,28 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
} }
} }
function fixVideoMetadataPerms2(file_path, type) {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
file_path_no_extension + '.info.json',
file_path_no_extension + ext + '.info.json',
// Thumbnails
file_path_no_extension + '.webp',
file_path_no_extension + '.jpg'
];
for (const file of files_to_fix) {
if (!fs.existsSync(file)) continue;
fs.chmodSync(file, 0o644);
}
}
function deleteJSONFile(name, type, customPath = null) { function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path'); : config_api.getConfigItem('ytdl_video_folder_path');
@@ -202,6 +256,18 @@ function deleteJSONFile(name, type, customPath = null) {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
} }
function deleteJSONFile2(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) { async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'}); let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) { if (!data) {
@@ -309,11 +375,15 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
module.exports = { module.exports = {
getJSONMp3: getJSONMp3, getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4, getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName, getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail, getDownloadedThumbnail: getDownloadedThumbnail,
getDownloadedThumbnail2: getDownloadedThumbnail2,
getExpectedFileSize: getExpectedFileSize, getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms, fixVideoMetadataPerms: fixVideoMetadataPerms,
fixVideoMetadataPerms2: fixVideoMetadataPerms2,
deleteJSONFile: deleteJSONFile, deleteJSONFile: deleteJSONFile,
deleteJSONFile2: deleteJSONFile2,
removeIDFromArchive, removeIDFromArchive, removeIDFromArchive, removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType, getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile, createContainerZipFile: createContainerZipFile,

16
src/app/components/downloads/downloads.component.html
View File

@@ -1,21 +1,21 @@
<div style="padding: 20px;"> <div style="padding: 20px;">
<div *ngFor="let session_downloads of downloads | keyvalue"> <div *ngFor="let session_downloads of downloads">
<ng-container *ngIf="keys(session_downloads.value).length > 0"> <ng-container *ngIf="keys(session_downloads).length > 2">
<mat-card style="padding-bottom: 30px; margin-bottom: 15px;"> <mat-card style="padding-bottom: 30px; margin-bottom: 15px;">
<h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads.key}} <h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads['session_id']}}
<span *ngIf="session_downloads.key === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span> <span *ngIf="session_downloads['session_id'] === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
</h4> </h4>
<div class="container"> <div class="container">
<div class="row"> <div class="row">
<div *ngFor="let download of session_downloads.value | keyvalue: sort_downloads; let i = index;" class="col-12 my-1"> <div *ngFor="let download of session_downloads | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
<mat-card *ngIf="download.value" class="mat-elevation-z3"> <mat-card *ngIf="download.key !== 'session_id' && download.key !== '_id' && download.value" class="mat-elevation-z3">
<app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads.key, download.value.uid)"></app-download-item> <app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads['session_id'], download.value.uid)"></app-download-item>
</mat-card> </mat-card>
</div> </div>
</div> </div>
</div> </div>
<div> <div>
<button style="top: 15px;" (click)="clearDownloads(session_downloads.key)" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button> <button style="top: 15px;" (click)="clearDownloads(session_downloads['session_id'])" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
</div> </div>
</mat-card> </mat-card>
</ng-container> </ng-container>

12
src/app/components/downloads/downloads.component.ts
View File

@@ -35,7 +35,7 @@ import { Router } from '@angular/router';
export class DownloadsComponent implements OnInit, OnDestroy { export class DownloadsComponent implements OnInit, OnDestroy {
downloads_check_interval = 1000; downloads_check_interval = 1000;
downloads = {}; downloads = [];
interval_id = null; interval_id = null;
keys = Object.keys; keys = Object.keys;
@@ -137,6 +137,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
this.downloads[session_id] = session_downloads_by_id; this.downloads[session_id] = session_downloads_by_id;
} else { } else {
for (let j = 0; j < session_download_ids.length; j++) { for (let j = 0; j < session_download_ids.length; j++) {
if (session_download_ids[j] === 'session_id' || session_download_ids[j] === '_id') continue;
const download_id = session_download_ids[j]; const download_id = session_download_ids[j];
const download = new_downloads_by_session[session_id][download_id] const download = new_downloads_by_session[session_id][download_id]
if (!this.downloads[session_id][download_id]) { if (!this.downloads[session_id][download_id]) {
@@ -156,11 +157,10 @@ export class DownloadsComponent implements OnInit, OnDestroy {
downloadsValid() { downloadsValid() {
let valid = false; let valid = false;
const keys = this.keys(this.downloads); for (let i = 0; i < this.downloads.length; i++) {
for (let i = 0; i < keys.length; i++) { const session_downloads = this.downloads[i];
const key = keys[i]; if (!session_downloads) continue;
const value = this.downloads[key]; if (this.keys(session_downloads).length > 2) {
if (this.keys(value).length > 0) {
valid = true; valid = true;
break; break;
} }

2
src/app/components/manage-role/manage-role.component.html
View File

@@ -5,7 +5,7 @@
<mat-list-item role="listitem" *ngFor="let permission of available_permissions"> <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
<h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3> <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
<span matLine> <span matLine>
<mat-radio-group [disabled]="permission === 'settings' && role.name === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission"> <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
<mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button> <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
<mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button> <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
</mat-radio-group> </mat-radio-group>

2
src/app/components/manage-role/manage-role.component.ts
View File

@@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit {
} }
changeRolePermissions(change, permission) { changeRolePermissions(change, permission) {
this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => { this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => {
if (res['success']) { if (res['success']) {
} else { } else {

2
src/app/components/modify-users/modify-users.component.html
View File

@@ -94,7 +94,7 @@
</div> </div>
<button color="primary" [matMenuTriggerFor]="edit_roles_menu" class="edit-role" mat-raised-button><ng-container i18n="Edit role">Edit Role</ng-container></button> <button color="primary" [matMenuTriggerFor]="edit_roles_menu" class="edit-role" mat-raised-button><ng-container i18n="Edit role">Edit Role</ng-container></button>
<mat-menu #edit_roles_menu="matMenu"> <mat-menu #edit_roles_menu="matMenu">
<button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.name}}</button> <button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.key}}</button>
</mat-menu> </mat-menu>
</div> </div>

11
src/app/components/modify-users/modify-users.component.ts
View File

@@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
getRoles() { getRoles() {
this.postsService.getRoles().subscribe(res => { this.postsService.getRoles().subscribe(res => {
this.roles = []; this.roles = res['roles'];
const roles = res['roles'];
const role_names = Object.keys(roles);
for (let i = 0; i < role_names.length; i++) {
const role_name = role_names[i];
this.roles.push({
name: role_name,
permissions: roles[role_name]['permissions']
});
}
}); });
} }

2
src/app/dialogs/modify-playlist/modify-playlist.component.ts
View File

@@ -61,7 +61,7 @@ export class ModifyPlaylistComponent implements OnInit {
} }
playlistChanged() { playlistChanged() {
return JSON.stringify(this.playlist) === JSON.stringify(this.original_playlist); return JSON.stringify(this.playlist) !== JSON.stringify(this.original_playlist);
} }
getPlaylist() { getPlaylist() {

13
src/app/main/main.component.html
View File

@@ -20,11 +20,16 @@
</ng-container> </ng-container>
</mat-label> </mat-label>
<mat-select [ngModelOptions]="{standalone: true}" [(ngModel)]="selectedQuality"> <mat-select [ngModelOptions]="{standalone: true}" [(ngModel)]="selectedQuality">
<ng-container *ngFor="let option of qualityOptions[(audioOnly) ? 'audio' : 'video']"> <mat-option [value]="''">
<mat-option *ngIf="option.value === '' || url && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats'] && cachedAvailableFormats[url]['formats'][(audioOnly) ? 'audio' : 'video'][option.value]" [value]="option.value"> Max
{{option.label}}
</mat-option> </mat-option>
</ng-container> <ng-container *ngIf="url && cachedAvailableFormats && cachedAvailableFormats[url]?.formats">
<ng-container *ngFor="let option of cachedAvailableFormats[url]['formats'][audioOnly ? 'audio' : 'video']">
<mat-option *ngIf="option.key !== 'best_audio_format'" [value]="option">
{{option.key}}
</mat-option>
</ng-container>
</ng-container>
</mat-select> </mat-select>
<div class="spinner-div" *ngIf="url !== '' && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats_loading']"> <div class="spinner-div" *ngIf="url !== '' && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats_loading']">
<mat-spinner [diameter]="25"></mat-spinner> <mat-spinner [diameter]="25"></mat-spinner>

56
src/app/main/main.component.ts
View File

@@ -500,23 +500,26 @@ export class MainComponent implements OnInit {
} }
getSelectedAudioFormat() { getSelectedAudioFormat() {
if (this.selectedQuality === '') { return null }; if (this.selectedQuality === '') { return null; }
const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats']; const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
if (cachedFormatsExists) { if (cachedFormatsExists) {
const audio_formats = this.cachedAvailableFormats[this.url]['formats']['audio']; const audio_formats = this.cachedAvailableFormats[this.url]['formats']['audio'];
return audio_formats[this.selectedQuality]['format_id']; return this.selectedQuality['format_id'];
} else { } else {
return null; return null;
} }
} }
getSelectedVideoFormat() { getSelectedVideoFormat() {
if (this.selectedQuality === '') { return null }; if (this.selectedQuality === '') { return null; }
const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats']; const cachedFormats = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
if (cachedFormatsExists) { if (cachedFormats) {
const video_formats = this.cachedAvailableFormats[this.url]['formats']['video']; const video_formats = cachedFormats['video'];
if (video_formats['best_audio_format'] && this.selectedQuality !== '') { if (this.selectedQuality) {
return video_formats[this.selectedQuality]['format_id'] + '+' + video_formats['best_audio_format']; let selected_video_format = this.selectedQuality['format_id'];
// add in audio format if necessary
if (!this.selectedQuality['acodec'] && cachedFormats['best_audio_format']) selected_video_format += `+${cachedFormats['best_audio_format']}`;
return selected_video_format;
} }
} }
return null; return null;
@@ -644,9 +647,8 @@ export class MainComponent implements OnInit {
this.errorFormats(url); this.errorFormats(url);
return; return;
} }
const parsed_infos = this.getAudioAndVideoFormats(infos.formats); this.cachedAvailableFormats[url]['formats'] = this.getAudioAndVideoFormats(infos.formats);
const available_formats = {audio: parsed_infos[0], video: parsed_infos[1]}; console.log(this.cachedAvailableFormats[url]['formats']);
this.cachedAvailableFormats[url]['formats'] = available_formats;
}, err => { }, err => {
this.errorFormats(url); this.errorFormats(url);
}); });
@@ -689,7 +691,7 @@ export class MainComponent implements OnInit {
if (audio_format) { if (audio_format) {
format_array.push('-f', audio_format); format_array.push('-f', audio_format);
} else if (this.selectedQuality) { } else if (this.selectedQuality) {
format_array.push('--audio-quality', this.selectedQuality); format_array.push('--audio-quality', this.selectedQuality['format_id']);
} }
// pushes formats // pushes formats
@@ -705,7 +707,7 @@ export class MainComponent implements OnInit {
if (video_format) { if (video_format) {
format_array = ['-f', video_format]; format_array = ['-f', video_format];
} else if (this.selectedQuality) { } else if (this.selectedQuality) {
format_array = [`bestvideo[height=${this.selectedQuality}]+bestaudio/best[height=${this.selectedQuality}]`]; format_array = [`bestvideo[height=${this.selectedQuality['format_id']}]+bestaudio/best[height=${this.selectedQuality}]`];
} }
// pushes formats // pushes formats
@@ -802,9 +804,11 @@ export class MainComponent implements OnInit {
} }
} }
getAudioAndVideoFormats(formats): any[] { getAudioAndVideoFormats(formats) {
const audio_formats = {}; const audio_formats: any = {};
const video_formats = {}; const video_formats: any = {};
console.log(formats);
for (let i = 0; i < formats.length; i++) { for (let i = 0; i < formats.length; i++) {
const format_obj = {type: null}; const format_obj = {type: null};
@@ -815,9 +819,12 @@ export class MainComponent implements OnInit {
format_obj.type = format_type; format_obj.type = format_type;
if (format_obj.type === 'audio' && format.abr) { if (format_obj.type === 'audio' && format.abr) {
const key = format.abr.toString() + 'K'; const key = format.abr.toString() + 'K';
format_obj['key'] = key;
format_obj['bitrate'] = format.abr; format_obj['bitrate'] = format.abr;
format_obj['format_id'] = format.format_id; format_obj['format_id'] = format.format_id;
format_obj['ext'] = format.ext; format_obj['ext'] = format.ext;
format_obj['label'] = key;
// don't overwrite if not m4a // don't overwrite if not m4a
if (audio_formats[key]) { if (audio_formats[key]) {
if (format.ext === 'm4a') { if (format.ext === 'm4a') {
@@ -828,11 +835,14 @@ export class MainComponent implements OnInit {
} }
} else if (format_obj.type === 'video') { } else if (format_obj.type === 'video') {
// check if video format is mp4 // check if video format is mp4
const key = format.format_note.replace('p', ''); const key = `${format.height}p${Math.round(format.fps)}`;
if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') { if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') {
format_obj['key'] = key;
format_obj['height'] = format.height; format_obj['height'] = format.height;
format_obj['acodec'] = format.acodec; format_obj['acodec'] = format.acodec;
format_obj['format_id'] = format.format_id; format_obj['format_id'] = format.format_id;
format_obj['label'] = key;
format_obj['fps'] = Math.round(format.fps);
// no acodec means no overwrite // no acodec means no overwrite
if (!(video_formats[key]) || format_obj['acodec'] !== 'none') { if (!(video_formats[key]) || format_obj['acodec'] !== 'none') {
@@ -842,9 +852,17 @@ export class MainComponent implements OnInit {
} }
} }
video_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats); const parsed_formats: any = {};
return [audio_formats, video_formats] parsed_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats);
parsed_formats['video'] = Object.values(video_formats);
parsed_formats['audio'] = Object.values(audio_formats);
parsed_formats['video'] = parsed_formats['video'].sort((a, b) => b.height - a.height || b.fps - a.fps);
parsed_formats['audio'] = parsed_formats['audio'].sort((a, b) => b.bitrate - a.bitrate);
return parsed_formats;
} }
getBestAudioFormatForMp4(audio_formats) { getBestAudioFormatForMp4(audio_formats) {

4
src/app/player/player.component.html
View File

@@ -27,11 +27,11 @@
</ng-container> </ng-container>
</div> </div>
<div class="col-2"> <div class="col-2">
<ng-container *ngIf="playlist.length > 1"> <ng-container *ngIf="db_playlist">
<button (click)="downloadContent()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button> <button (click)="downloadContent()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button> <button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
</ng-container> </ng-container>
<ng-container *ngIf="playlist.length === 1"> <ng-container *ngIf="db_file">
<button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button> <button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button> <button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
<button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button> <button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button>

9
src/app/player/player.component.ts
View File

@@ -171,7 +171,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
console.error('Failed to increment view count'); console.error('Failed to increment view count');
console.error(err); console.error(err);
}); });
this.uids = this.db_file['uid']; this.uids = [this.db_file['uid']];
this.show_player = true; this.show_player = true;
this.parseFileNames(); this.parseFileNames();
} }
@@ -304,12 +304,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
} }
downloadContent() { downloadContent() {
const fileNames = []; const zipName = this.db_playlist.name;
for (let i = 0; i < this.playlist.length; i++) {
fileNames.push(this.playlist[i].title);
}
const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0];
this.downloading = true; this.downloading = true;
this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => { this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => {
this.downloading = false; this.downloading = false;

17
src/app/posts.services.ts
View File

@@ -184,6 +184,18 @@ export class PostsService implements CanActivate {
cropFileSettings: cropFileSettings}, this.httpOptions); cropFileSettings: cropFileSettings}, this.httpOptions);
} }
getDBInfo() {
return this.http.post(this.path + 'getDBInfo', {}, this.httpOptions);
}
transferDB(local_to_remote) {
return this.http.post(this.path + 'transferDB', {local_to_remote: local_to_remote}, this.httpOptions);
}
testConnectionString() {
return this.http.post(this.path + 'testConnectionString', {}, this.httpOptions);
}
killAllDownloads() { killAllDownloads() {
return this.http.post(this.path + 'killAllDownloads', {}, this.httpOptions); return this.http.post(this.path + 'killAllDownloads', {}, this.httpOptions);
} }
@@ -236,11 +248,12 @@ export class PostsService implements CanActivate {
return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions); return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions);
} }
downloadFileFromServer(uid, uuid = null, sub_id = null) { downloadFileFromServer(uid, uuid = null, sub_id = null, is_playlist = null) {
return this.http.post(this.path + 'downloadFileFromServer', { return this.http.post(this.path + 'downloadFileFromServer', {
uid: uid, uid: uid,
uuid: uuid, uuid: uuid,
sub_id: sub_id sub_id: sub_id,
is_playlist: is_playlist
}, },
{responseType: 'blob', params: this.httpOptions.params}); {responseType: 'blob', params: this.httpOptions.params});
} }

37
src/app/settings/settings.component.html
View File

@@ -280,6 +280,43 @@
</div> </div>
</ng-template> </ng-template>
</mat-tab> </mat-tab>
<!-- Database -->
<mat-tab label="Database" i18n-label="Database settings label">
<ng-template matTabContent>
<div *ngIf="new_config" class="container-fluid">
<div class="row">
<div class="col-12 mt-3">
<div *ngIf="db_info">
<h5 i18n="Database info title">Database Info</h5>
<p><ng-container i18n="Database location label">Database location:</ng-container>&nbsp;<strong>{{db_info['using_local_db'] ? 'Local' : 'MongoDB'}}</strong></p>
<h6 i18n="Records per table label">Records per table</h6>
<mat-list style="padding-top: 0px">
<mat-list-item style="height: 28px" *ngFor="let table_stats of db_info['stats_by_table'] | keyvalue">
{{table_stats.key}}: {{table_stats.value.records_count}}
</mat-list-item>
</mat-list>
<mat-form-field style="width: 100%; margin-top: 15px; margin-bottom: 10px" color="accent">
<input [(ngModel)]="new_config['Database']['mongodb_connection_string']" matInput placeholder="MongoDB Connection String" i18n-placeholder="MongoDB Connection String" required>
<mat-hint><ng-container i18n="MongoDB Connection String setting hint AKA preamble">Example:</ng-container>&nbsp;mongodb://127.0.0.1:27017/?compressors=zlib</mat-hint>
</mat-form-field>
<br>
<button (click)="testConnectionString()" [disabled]="testing_connection_string" mat-flat-button color="accent"><ng-container i18n="Test connection string button">Test connection string</ng-container></button>
<br>
<button class="transfer-db-button" [disabled]="db_transferring" color="accent" (click)="transferDB()" mat-raised-button><ng-container i18n="Transfer DB button">Transfer DB to </ng-container>{{db_info['using_local_db'] ? 'MongoDB' : 'Local'}}</button>
</div>
<div *ngIf="!db_info">
<ng-container i18n="Database info not retrieved error message">Database information could not be retrieved. Check the server logs for more information.</ng-container>
</div>
</div>
</div>
</div>
</ng-template>
</mat-tab>
<!-- Advanced --> <!-- Advanced -->
<mat-tab label="Advanced" i18n-label="Host settings label"> <mat-tab label="Advanced" i18n-label="Host settings label">
<ng-template matTabContent> <ng-template matTabContent>

13
src/app/settings/settings.component.scss
View File

@@ -77,8 +77,13 @@
} }
.category-custom-placeholder { .category-custom-placeholder {
background: #ccc; background: #ccc;
border: dotted 3px #999; border: dotted 3px #999;
min-height: 60px; min-height: 60px;
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
}
.transfer-db-button {
margin-top: 10px;
margin-bottom: 10px;
} }

59
src/app/settings/settings.component.ts
View File

@@ -29,6 +29,10 @@ export class SettingsComponent implements OnInit {
generated_bookmarklet_code = null; generated_bookmarklet_code = null;
bookmarkletAudioOnly = false; bookmarkletAudioOnly = false;
db_info = null;
db_transferring = false;
testing_connection_string = false;
_settingsSame = true; _settingsSame = true;
latestGithubRelease = null; latestGithubRelease = null;
@@ -48,6 +52,7 @@ export class SettingsComponent implements OnInit {
ngOnInit() { ngOnInit() {
this.getConfig(); this.getConfig();
this.getDBInfo();
this.generated_bookmarklet_code = this.sanitizer.bypassSecurityTrustUrl(this.generateBookmarkletCode()); this.generated_bookmarklet_code = this.sanitizer.bypassSecurityTrustUrl(this.generateBookmarkletCode());
@@ -263,6 +268,60 @@ export class SettingsComponent implements OnInit {
}); });
} }
getDBInfo() {
this.postsService.getDBInfo().subscribe(res => {
this.db_info = res['db_info'];
});
}
transferDB() {
const dialogRef = this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: 'Transfer DB',
dialogText: `Are you sure you want to transfer the DB?`,
submitText: 'Transfer',
}
});
dialogRef.afterClosed().subscribe(confirmed => {
if (confirmed) {
this._transferDB();
}
});
}
_transferDB() {
this.db_transferring = true;
this.postsService.transferDB(this.db_info['using_local_db']).subscribe(res => {
this.db_transferring = false;
const success = res['success'];
if (success) {
this.openSnackBar('Successfully transfered DB! Reloading info...');
this.getDBInfo();
} else {
this.openSnackBar('Failed to transfer DB -- transfer was aborted. Error: ' + res['error']);
}
}, err => {
this.db_transferring = false;
this.openSnackBar('Failed to transfer DB -- API call failed. See browser logs for details.');
console.error(err);
});
}
testConnectionString() {
this.testing_connection_string = true;
this.postsService.testConnectionString().subscribe(res => {
this.testing_connection_string = false;
if (res['success']) {
this.postsService.openSnackBar('Connection successful!');
} else {
this.postsService.openSnackBar('Connection failed! Error: ' + res['error']);
}
}, err => {
this.testing_connection_string = false;
this.postsService.openSnackBar('Connection failed! Error: Server error. See logs for more info.');
});
}
// snackbar helper // snackbar helper
public openSnackBar(message: string, action: string = '') { public openSnackBar(message: string, action: string = '') {
this.snackBar.open(message, action, { this.snackBar.open(message, action, {