TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Changed DB structure again

Browse Source 
Added support for MongoDB

Added tests relating to new DB system

Category rules are now case insensitive

Fixed playlist modification change state
This commit is contained in:
Isaac Abadi
2021-07-16 00:05:08 -06:00
parent 984e990103
commit bb6503e86d
27 changed files with 1426 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
backend/app.js
View File

@@ -79,9 +79,9 @@ const logger = winston.createLogger({
});
config_api.initialize(logger);
auth_api.initialize(db, users_db, logger);
db_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api);
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
categories_api.initialize(db, users_db, logger, db_api);
// Set some defaults
@@ -183,7 +183,7 @@ if (writeConfigMode) {
loadConfig();
}
var downloads = {};
var downloads = [];
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
@@ -210,10 +210,12 @@ async function checkMigrations() {
if (!simplified_db_migration_complete) {
logger.info('Beginning migration: 4.1->4.2+')
let success = await simplifyDBFileStructure();
success = success && await addMetadataPropertyToDB('view_count');
success = success && await addMetadataPropertyToDB('description');
success = success && await addMetadataPropertyToDB('height');
success = success && await addMetadataPropertyToDB('abr');
success = success && await db_api.addMetadataPropertyToDB('view_count');
success = success && await db_api.addMetadataPropertyToDB('description');
success = success && await db_api.addMetadataPropertyToDB('height');
success = success && await db_api.addMetadataPropertyToDB('abr');
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
if (success) { logger.info('4.1->4.2+ migration complete!'); }
else { logger.error('Migration failed: 4.1->4.2+'); }
}
@@ -290,28 +292,6 @@ async function simplifyDBFileStructure() {
return true;
}
async function addMetadataPropertyToDB(property_key) {
try {
const dirs_to_check = db_api.getFileDirectoriesAndDBs();
for (const dir_to_check of dirs_to_check) {
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (const file of files) {
if (file[property_key]) {
dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write();
}
}
}
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
return true;
} catch(err) {
logger.error(err);
return false;
}
}
async function startServer() {
if (process.env.USING_HEROKU && process.env.PORT) {
// default to heroku port if using heroku
@@ -612,6 +592,9 @@ async function setConfigFromEnv() {
async function loadConfig() {
loadConfigValues();
// connect to DB
await db_api.connectToDB();
// creates archive path if missing
await fs.ensureDir(archivePath);
@@ -624,7 +607,7 @@ async function loadConfig() {
// get subscriptions
if (allowSubscriptions) {
// set downloading to false
let subscriptions = subscriptions_api.getAllSubscriptions();
let subscriptions = await subscriptions_api.getAllSubscriptions();
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
// runs initially, then runs every ${subscriptionCheckInterval} seconds
watchSubscriptions();
@@ -633,10 +616,10 @@ async function loadConfig() {
}, subscriptionsCheckInterval * 1000);
}
db_api.importUnregisteredFiles();
await db_api.importUnregisteredFiles();
// load in previous downloads
downloads = db.get('downloads').value();
downloads = await db_api.getRecords('downloads');
// start the server here
startServer();
@@ -684,7 +667,7 @@ function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
}
async function watchSubscriptions() {
let subscriptions = subscriptions_api.getAllSubscriptions();
let subscriptions = await subscriptions_api.getAllSubscriptions();
if (!subscriptions) return;
@@ -903,8 +886,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
// adds download to download helper
const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared';
if (!downloads[session]) downloads[session] = {};
downloads[session][download_uid] = {
let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
if (!session_downloads) {
session_downloads = {session_id: session};
downloads.push(session_downloads);
}
session_downloads[download_uid] = {
uid: download_uid,
ui_uid: options.ui_uid,
downloading: true,
@@ -916,7 +903,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
timestamp_start: Date.now(),
filesize: null
};
const download = downloads[session][download_uid];
const download = session_downloads[download_uid];
updateDownloads();
let download_checker = null;
@@ -1027,7 +1014,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
}
// registers file in DB
const file_obj = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings);
const file_obj = await db_api.registerFileDB2(full_file_path, options.user, category, null, options.cropFileSettings);
// TODO: remove the following line
if (file_name) file_names.push(file_name);
@@ -1070,146 +1057,6 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
});
}
async function downloadFileByURL_normal(url, type, options, sessionID = null) {
return new Promise(async resolve => {
var date = Date.now();
var file_uid = null;
const is_audio = type === 'audio';
const ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; }
// prepend with user if needed
let multiUserMode = null;
if (options.user) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, options.user, type);
fs.ensureDirSync(user_path);
fileFolderPath = user_path + path.sep;
multiUserMode = {
user: options.user,
file_path: fileFolderPath
}
options.customFileFolderPath = fileFolderPath;
}
options.downloading_method = 'normal';
const downloadConfig = await generateArgs(url, type, options);
// adds download to download helper
const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared';
if (!downloads[session]) downloads[session] = {};
downloads[session][download_uid] = {
uid: download_uid,
ui_uid: options.ui_uid,
downloading: true,
complete: false,
url: url,
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now()
};
const download = downloads[session][download_uid];
updateDownloads();
const video = youtubedl(url,
// Optional arguments passed to youtube-dl.
downloadConfig,
// Additional options can be given for calling `child_process.execFile()`.
{ cwd: __dirname });
let video_info = null;
let file_size = 0;
// Will be called when the download starts.
video.on('info', function(info) {
video_info = info;
file_size = video_info.size;
const json_path = utils.removeFileExtension(video_info._filename) + '.info.json';
fs.ensureFileSync(json_path);
fs.writeJSONSync(json_path, video_info);
video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' }))
});
// Will be called if download was already completed and there is nothing more to download.
video.on('complete', function complete(info) {
'use strict'
logger.info('file ' + info._filename + ' already downloaded.')
})
let download_pos = 0;
video.on('data', function data(chunk) {
download_pos += chunk.length
// `size` should not be 0 here.
if (file_size) {
let percent = (download_pos / file_size * 100).toFixed(2)
download['percent_complete'] = percent;
}
});
video.on('end', async function() {
let new_date = Date.now();
let difference = (new_date - date)/1000;
logger.debug(`Video download delay: ${difference} seconds.`);
download['timestamp_end'] = Date.now();
download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext];
download['complete'] = true;
updateDownloads();
// audio-only cleanup
if (is_audio) {
// filename fix
video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3';
// ID3 tagging
let tags = {
title: video_info['title'],
artist: video_info['artist'] ? video_info['artist'] : video_info['uploader']
}
let success = NodeID3.write(tags, video_info._filename);
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename);
const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm';
const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4';
// check if audio file is webm
if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']);
else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']);
}
// registers file in DB
const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length);
file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode);
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name));
resolve({
encodedPath: videopathEncoded,
file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready
uid: file_uid
});
});
video.on('error', function error(err) {
logger.error(err);
download[error] = err;
updateDownloads();
resolve(false);
});
});
}
async function generateArgs(url, type, options) {
var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
var globalArgs = config_api.getConfigItem('ytdl_custom_args');
@@ -1250,8 +1097,8 @@ async function generateArgs(url, type, options) {
qualityPath = ['-f', customQualityConfiguration];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (maxBitrate && is_audio) {
qualityPath = ['--audio-quality', maxBitrate]
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
if (customOutput) {
@@ -1401,23 +1248,6 @@ async function getUrlInfos(urls) {
// ffmpeg helper functions
async function convertFileToMp3(input_file, output_file) {
logger.verbose(`Converting ${input_file} to ${output_file}...`);
return new Promise(resolve => {
ffmpeg(input_file).noVideo().toFormat('mp3')
.on('end', () => {
logger.verbose(`Conversion for '${output_file}' complete.`);
fs.unlinkSync(input_file)
resolve(true);
})
.on('error', (err) => {
logger.error('Failed to convert audio file to the correct format.');
logger.error(err);
resolve(false);
}).save(output_file);
});
}
async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
@@ -1456,8 +1286,9 @@ async function writeToBlacklist(type, line) {
// download management functions
function updateDownloads() {
db.assign({downloads: downloads}).write();
async function updateDownloads() {
await db_api.removeAllRecords('downloads');
if (downloads.length !== 0) await db_api.insertRecordsIntoTable('downloads', downloads);
}
function checkDownloadPercent(download) {
@@ -1489,7 +1320,6 @@ function checkDownloadPercent(download) {
}
});
download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
updateDownloads();
});
}
@@ -1715,6 +1545,37 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
res.send({success: true});
});
app.post('/api/getDBInfo', optionalJwt, async (req, res) => {
const db_info = await db_api.getDBStats();
res.send({db_info: db_info});
});
app.post('/api/transferDB', optionalJwt, async (req, res) => {
const local_to_remote = req.body.local_to_remote;
let success = null;
let error = '';
if (local_to_remote === config_api.getConfigItem('ytdl_use_local_db')) {
success = await db_api.transferDB(local_to_remote);
if (!success) error = 'Unknown error';
else config_api.setConfigItem('ytdl_use_local_db', !local_to_remote);
} else {
success = false;
error = `Failed to transfer DB as it cannot transition into its current status: ${local_to_remote ? 'MongoDB' : 'Local DB'}`;
logger.error(error);
}
res.send({success: success, error: error});
});
app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
let success = null;
let error = '';
success = await db_api.connectToDB(5, true);
if (!success) error = 'Connection string failed.';
res.send({success: success, error: error});
});
app.post('/api/downloadFile', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos
const url = req.body.url;
@@ -1731,15 +1592,7 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
cropFileSettings: req.body.cropFileSettings
}
const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload();
if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.');
const is_playlist = url.includes('playlist');
let result_obj = null;
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
let result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID);
if (result_obj) {
res.send(result_obj);
} else {
@@ -1752,29 +1605,17 @@ app.post('/api/killAllDownloads', optionalJwt, async function(req, res) {
res.send(result_obj);
});
/**
* add thumbnails if present
* @param files - List of files with thumbnailPath property.
*/
async function addThumbnails(files) {
await Promise.all(files.map(async file => {
const thumbnailPath = file['thumbnailPath'];
if (thumbnailPath && (await fs.pathExists(thumbnailPath))) {
file['thumbnailBlob'] = await fs.readFile(thumbnailPath);
}
}));
}
// gets all download mp3s
app.get('/api/getMp3s', optionalJwt, async function(req, res) {
var mp3s = db.get('files').value().filter(file => file.isAudio === true);
var playlists = db.get('playlists').value();
// TODO: simplify
let mp3s = await db_api.getRecords('files', {isAudio: true});
let playlists = await db_api.getRecords('playlists');
const is_authenticated = req.isAuthenticated();
if (is_authenticated) {
// get user audio files/playlists
auth_api.passport.authenticate('jwt')
mp3s = auth_api.getUserVideos(req.user.uid, 'audio');
playlists = auth_api.getUserPlaylists(req.user.uid);
mp3s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: true});
playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
}
mp3s = JSON.parse(JSON.stringify(mp3s));
@@ -1787,15 +1628,15 @@ app.get('/api/getMp3s', optionalJwt, async function(req, res) {
// gets all download mp4s
app.get('/api/getMp4s', optionalJwt, async function(req, res) {
var mp4s = db.get('files').value().filter(file => file.isAudio === false);
var playlists = db.get('playlists').value();
let mp4s = await db_api.getRecords('files', {isAudio: false});
let playlists = await db_api.getRecords('playlists');
const is_authenticated = req.isAuthenticated();
if (is_authenticated) {
// get user videos/playlists
auth_api.passport.authenticate('jwt')
mp4s = auth_api.getUserVideos(req.user.uid, 'video');
playlists = auth_api.getUserPlaylists(req.user.uid);
mp4s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: false});
playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
}
mp4s = JSON.parse(JSON.stringify(mp4s));
@@ -1806,20 +1647,14 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) {
});
});
app.post('/api/getFile', optionalJwt, function (req, res) {
app.post('/api/getFile', optionalJwt, async function (req, res) {
var uid = req.body.uid;
var type = req.body.type;
var uuid = req.body.uuid;
var file = null;
var file = await db_api.getRecord('files', {uid: uid});
if (req.isAuthenticated()) {
file = auth_api.getUserVideo(req.user.uid, uid);
} else if (uuid) {
file = auth_api.getUserVideo(uuid, uid, true);
} else {
file = db.get('files').find({uid: uid}).value();
}
if (uuid && !file['sharingEnabled']) file = null;
// check if chat exists for twitch videos
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
@@ -1842,18 +1677,12 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
let playlists = null;
const uuid = req.isAuthenticated() ? req.user.uid : null;
let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : [];
let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (await subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : [];
// get basic info depending on multi-user mode being enabled
if (uuid) {
files = auth_api.getUserVideos(req.user.uid);
playlists = auth_api.getUserPlaylists(req.user.uid, files);
} else {
files = db.get('files').value();
playlists = JSON.parse(JSON.stringify(db.get('playlists').value()));
}
files = await db_api.getRecords('files', {user_uid: uuid});
playlists = await db_api.getRecords('playlists', {user_uid: uuid});
const categories = categories_api.getCategoriesAsPlaylists(files);
const categories = await categories_api.getCategoriesAsPlaylists(files);
if (categories) {
playlists = playlists.concat(categories);
}
@@ -1872,11 +1701,6 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
files = JSON.parse(JSON.stringify(files));
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
// add thumbnails if present
// await addThumbnails(files);
}
res.send({
files: files,
playlists: playlists
@@ -1952,7 +1776,7 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
});
// video sharing
app.post('/api/enableSharing', optionalJwt, function(req, res) {
app.post('/api/enableSharing', optionalJwt, async (req, res) => {
var uid = req.body.uid;
var is_playlist = req.body.is_playlist;
let success = false;
@@ -1968,15 +1792,9 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) {
try {
success = true;
if (!is_playlist) {
db.get(`files`)
.find({uid: uid})
.assign({sharingEnabled: true})
.write();
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: true})
} else if (is_playlist) {
db.get(`playlists`)
.find({id: uid})
.assign({sharingEnabled: true})
.write();
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true});
} else if (false) {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
@@ -1995,7 +1813,7 @@ app.post('/api/enableSharing', optionalJwt, function(req, res) {
});
});
app.post('/api/disableSharing', optionalJwt, function(req, res) {
app.post('/api/disableSharing', optionalJwt, async function(req, res) {
var type = req.body.type;
var uid = req.body.uid;
var is_playlist = req.body.is_playlist;
@@ -2012,15 +1830,9 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) {
try {
success = true;
if (!is_playlist && type !== 'subscription') {
db.get(`files`)
.find({uid: uid})
.assign({sharingEnabled: false})
.write();
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
} else if (is_playlist) {
db.get(`playlists`)
.find({id: uid})
.assign({sharingEnabled: false})
.write();
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
} else if (type === 'subscription') {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
@@ -2062,7 +1874,7 @@ app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
// categories
app.post('/api/getAllCategories', optionalJwt, async (req, res) => {
const categories = db.get('categories').value();
const categories = await db_api.getRecords('categories');
res.send({categories: categories});
});
@@ -2075,7 +1887,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => {
custom_output: ''
};
db.get('categories').push(new_category).write();
await db_api.insertRecordIntoTable('categories', new_category);
res.send({
new_category: new_category,
@@ -2086,7 +1898,7 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => {
app.post('/api/deleteCategory', optionalJwt, async (req, res) => {
const category_uid = req.body.category_uid;
db.get('categories').remove({uid: category_uid}).write();
await db_api.removeRecord('categories', {uid: category_uid});
res.send({
success: true
@@ -2095,13 +1907,14 @@ app.post('/api/deleteCategory', optionalJwt, async (req, res) => {
app.post('/api/updateCategory', optionalJwt, async (req, res) => {
const category = req.body.category;
db.get('categories').find({uid: category.uid}).assign(category).write();
await db_api.updateRecord('categories', {uid: category.uid}, category)
res.send({success: true});
});
app.post('/api/updateCategories', optionalJwt, async (req, res) => {
const categories = req.body.categories;
db.get('categories').assign(categories).write();
await db_api.removeAllRecords('categories');
await db_api.insertRecordsIntoTable('categories', categories);
res.send({success: true});
});
@@ -2200,9 +2013,9 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
// get sub from db
let subscription = null;
if (subID) {
subscription = subscriptions_api.getSubscription(subID, user_uid)
subscription = await subscriptions_api.getSubscription(subID, user_uid)
} else if (subName) {
subscription = subscriptions_api.getSubscriptionByName(subName, user_uid)
subscription = await subscriptions_api.getSubscriptionByName(subName, user_uid)
}
if (!subscription) {
@@ -2213,7 +2026,8 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
// get sub videos
if (subscription.name && !subscription.streamingOnly) {
var parsed_files = subscription.videos;
var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
subscription['videos'] = parsed_files;
if (!parsed_files) {
parsed_files = [];
let base_path = null;
@@ -2310,7 +2124,7 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
let user_uid = req.isAuthenticated() ? req.user.uid : null;
// get subs from api
let subscriptions = subscriptions_api.getSubscriptions(user_uid);
let subscriptions = await subscriptions_api.getSubscriptions(user_uid);
res.send({
subscriptions: subscriptions
@@ -2343,7 +2157,8 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await db_api.getVideo(uid, uuid);
file_objs.push(file_obj);
if (file_obj) file_objs.push(file_obj);
// TODO: remove file from playlist if could not be found
}
}
@@ -2364,10 +2179,7 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => {
if (req.isAuthenticated()) {
auth_api.updatePlaylistFiles(req.user.uid, playlistID, uids);
} else {
db.get(`playlists`)
.find({id: playlistID})
.assign({uids: uids})
.write();
await db_api.updateRecord('playlists', {id: playlistID}, {uids: uids})
}
success = true;
@@ -2393,14 +2205,8 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
let success = null;
try {
if (req.isAuthenticated()) {
auth_api.removePlaylist(req.user.uid, playlistID);
} else {
// removes playlist from playlists
db.get(`playlists`)
.remove({id: playlistID})
.write();
}
// removes playlist from playlists
await db_api.removeRecord('playlists', {id: playlistID})
success = true;
} catch(e) {
@@ -2611,20 +2417,14 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
});
app.post('/api/download', async (req, res) => {
var session_id = req.body.session_id;
var download_id = req.body.download_id;
const session_id = req.body.session_id;
const download_id = req.body.download_id;
const session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session_id);
let found_download = null;
// find download
if (downloads[session_id] && Object.keys(downloads[session_id])) {
let session_downloads = Object.values(downloads[session_id]);
for (let i = 0; i < session_downloads.length; i++) {
let session_download = session_downloads[i];
if (session_download && session_download['ui_uid'] === download_id) {
found_download = session_download;
break;
}
}
if (session_downloads && Object.keys(session_downloads)) {
found_download = Object.values(session_downloads).find(session_download => session_download['ui_uid'] === download_id);
}
if (found_download) {
@@ -2642,26 +2442,22 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
var download_id = req.body.download_id;
if (delete_all) {
// delete all downloads
downloads = {};
downloads = [];
success = true;
} else if (download_id) {
// delete just 1 download
if (downloads[session_id][download_id]) {
delete downloads[session_id][download_id];
const session_downloads = downloads.find(session => session['session_id'] === session_id);
if (session_downloads && session_downloads[download_id]) {
delete session_downloads[download_id];
success = true;
} else if (!downloads[session_id]) {
} else if (!session_downloads) {
logger.error(`Session ${session_id} has no downloads.`)
} else if (!downloads[session_id][download_id]) {
} else if (!session_downloads[download_id]) {
logger.error(`Download '${download_id}' for session '${session_id}' could not be found`);
}
} else if (session_id) {
// delete a session's downloads
if (downloads[session_id]) {
delete downloads[session_id];
success = true;
} else {
logger.error(`Session ${session_id} has no downloads.`)
}
downloads = downloads.filter(session => session['session_id'] !== session_id);
}
updateDownloads();
res.send({success: success, downloads: downloads});
@@ -2746,29 +2542,28 @@ app.post('/api/auth/changePassword', optionalJwt, async (req, res) => {
res.send({success: success});
});
app.post('/api/auth/adminExists', async (req, res) => {
let exists = auth_api.adminExists();
let exists = await auth_api.adminExists();
res.send({exists: exists});
});
// user management
app.post('/api/getUsers', optionalJwt, async (req, res) => {
let users = users_db.get('users').value();
let users = await db_api.getRecords('users');
res.send({users: users});
});
app.post('/api/getRoles', optionalJwt, async (req, res) => {
let roles = users_db.get('roles').value();
let roles = await db_api.getRecords('roles');
res.send({roles: roles});
});
app.post('/api/updateUser', optionalJwt, async (req, res) => {
let change_obj = req.body.change_object;
try {
const user_db_obj = users_db.get('users').find({uid: change_obj.uid});
if (change_obj.name) {
user_db_obj.assign({name: change_obj.name}).write();
await db_api.updateRecord('users', {uid: change_obj.uid}, {name: change_obj.name});
}
if (change_obj.role) {
user_db_obj.assign({role: change_obj.role}).write();
await db_api.updateRecord('users', {uid: change_obj.uid}, {role: change_obj.role});
}
res.send({success: true});
} catch (err) {
@@ -2780,13 +2575,17 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
app.post('/api/deleteUser', optionalJwt, async (req, res) => {
let uid = req.body.uid;
try {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = users_db.get('users').find({uid: uid});
if (user_db_obj.value()) {
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
users_db.get('users').remove({uid: uid}).write();
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
}
res.send({success: true});
} catch (err) {
@@ -2805,7 +2604,7 @@ app.post('/api/changeUserPermissions', optionalJwt, async (req, res) => {
return;
}
const success = auth_api.changeUserPermissions(user_uid, permission, new_value);
const success = await auth_api.changeUserPermissions(user_uid, permission, new_value);
res.send({success: success});
});
@@ -2820,7 +2619,7 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
return;
}
const success = auth_api.changeRolePermissions(role, permission, new_value);
const success = await auth_api.changeRolePermissions(role, permission, new_value);
res.send({success: success});
});

4
backend/appdata/default.json
View File

@@ -54,6 +54,10 @@
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,

138
backend/authentication/auth.js
View File

@@ -13,16 +13,15 @@ var JwtStrategy = require('passport-jwt').Strategy,
// other required vars
let logger = null;
let db = null;
let users_db = null;
let db_api = null;
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function(input_db, input_users_db, input_logger) {
exports.initialize = function(db_api, input_logger) {
setLogger(input_logger)
setDB(input_db, input_users_db);
setDB(db_api);
/*************************
* Authentication module
@@ -32,21 +31,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
SERVER_SECRET = null;
if (users_db.get('jwt_secret').value()) {
SERVER_SECRET = users_db.get('jwt_secret').value();
if (db_api.users_db.get('jwt_secret').value()) {
SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
} else {
SERVER_SECRET = uuid();
users_db.set('jwt_secret', SERVER_SECRET).write();
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
/*opts.issuer = 'example.com';
opts.audience = 'example.com';*/
exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) {
const user = users_db.get('users').find({uid: jwt_payload.user}).value();
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
} else {
@@ -60,9 +57,8 @@ function setLogger(input_logger) {
logger = input_logger;
}
function setDB(input_db, input_users_db) {
db = input_db;
users_db = input_users_db;
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport');
@@ -78,7 +74,7 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = function(req, res) {
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
@@ -96,20 +92,20 @@ exports.registerUser = function(req, res) {
}
bcrypt.hash(plaintextPassword, saltRounds)
.then(function(hash) {
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (users_db.get('users').find({uid: userid}).value()) {
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (users_db.get('users').find({name: username}).value()) {
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
@@ -143,7 +139,7 @@ exports.registerUser = function(req, res) {
exports.login = async (username, password) => {
const user = users_db.get('users').find({name: username}).value();
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); false }
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
@@ -164,17 +160,17 @@ var getLDAPConfiguration = function(req, callback) {
};
exports.passport.use(new LdapStrategy(getLDAPConfiguration,
function(user, done) {
async function(user, done) {
// check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false);
const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value();
let db_user = await db_api.getRecord('users', {uid: user_uid});
if (!db_user) {
// generate DB user
let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
db_user = new_user;
logger.verbose(`Generated new user ${user_uid} using LDAP`);
}
@@ -198,11 +194,11 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.returnAuthResponse = function(req, res) {
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
token: req.token,
permissions: exports.userPermissions(req.user.uid),
permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -215,7 +211,7 @@ exports.returnAuthResponse = function(req, res) {
* It also passes the user object to the next
* middleware through res.locals
**************************************/
exports.ensureAuthenticatedElseError = function(req, res, next) {
exports.ensureAuthenticatedElseError = (req, res, next) => {
var token = getToken(req.query);
if( token ) {
try {
@@ -233,10 +229,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
}
// change password
exports.changeUserPassword = async function(user_uid, new_pass) {
exports.changeUserPassword = async (user_uid, new_pass) => {
try {
const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
return true;
} catch (err) {
return false;
@@ -244,16 +240,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
}
// change user permissions
exports.changeUserPermissions = function(user_uid, permission, new_value) {
exports.changeUserPermissions = async (user_uid, permission, new_value) => {
try {
const user_db_obj = users_db.get('users').find({uid: user_uid});
user_db_obj.get('permissions').pull(permission).write();
user_db_obj.get('permission_overrides').pull(permission).write();
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
if (new_value === 'yes') {
user_db_obj.get('permissions').push(permission).write();
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} else if (new_value === 'no') {
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
}
return true;
} catch (err) {
@@ -263,12 +258,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
}
// change role permissions
exports.changeRolePermissions = function(role, permission, new_value) {
exports.changeRolePermissions = async (role, permission, new_value) => {
try {
const role_db_obj = users_db.get('roles').get(role);
role_db_obj.get('permissions').pull(permission).write();
await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
if (new_value === 'yes') {
role_db_obj.get('permissions').push(permission).write();
await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
}
return true;
} catch (err) {
@@ -277,19 +271,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
}
}
exports.adminExists = function() {
return !!users_db.get('users').find({uid: 'admin'}).value();
exports.adminExists = async function() {
return !!(await db_api.getRecord('users', {uid: 'admin'}));
}
// video stuff
exports.getUserVideos = function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value();
return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files'];
exports.getUserVideos = async function(user_uid, type) {
const files = await db_api.getRecords('files', {user_uid: user_uid});
return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
}
exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -302,19 +296,17 @@ exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
return true;
}
exports.removePlaylist = function(user_uid, playlistID) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write();
exports.removePlaylist = async function(user_uid, playlistID) {
await db_api.removeRecord('playlist', {playlistID: playlistID});
return true;
}
exports.getUserPlaylists = function(user_uid, user_files = null) {
const user = users_db.get('users').find({uid: user_uid}).value();
const playlists = JSON.parse(JSON.stringify(user['playlists']));
return playlists;
exports.getUserPlaylists = async function(user_uid, user_files = null) {
return await db_api.getRecords('playlists', {user_uid: user_uid});
}
exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) {
let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value();
exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
let playlist = await db_api.getRecord('playlists', {id: playlistID});
// prevent unauthorized users from accessing the file info
if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -322,40 +314,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false)
return playlist;
}
exports.registerUserFile = function(user_uid, file_object) {
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
path: file_object['path']
}).write();
users_db.get('users').find({uid: user_uid}).get(`files`)
.push(file_object)
.write();
}
exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid});
if (user_db_obj.value()) {
const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid});
if (file_db_obj.value()) {
success = true;
file_db_obj.assign({sharingEnabled: enabled}).write();
}
}
is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
success = true;
return success;
}
exports.userHasPermission = function(user_uid, permission) {
const user_obj = users_db.get('users').find({uid: user_uid}).value();
exports.userHasPermission = async function(user_uid, permission) {
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return false;
}
const role_permissions = (users_db.get('roles').value())['permissions'];
const role_permissions = (await db_api.getRecords('roles'))['permissions'];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -378,16 +353,17 @@ exports.userHasPermission = function(user_uid, permission) {
}
}
exports.userPermissions = function(user_uid) {
exports.userPermissions = async function(user_uid) {
let user_permissions = [];
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return null;
}
const role_permissions = users_db.get('roles').get(role).get('permissions').value()
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];

17
backend/categories.js
View File

@@ -39,10 +39,9 @@ async function categorize(file_jsons) {
if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
let selected_category = null;
const categories = getCategories();
const categories = await getCategories();
if (!categories) {
logger.warn('Categories could not be found. Initializing categories...');
db.assign({categories: []}).write();
logger.warn('Categories could not be found.');
return null;
}
@@ -64,14 +63,14 @@ async function categorize(file_jsons) {
return selected_category;
}
function getCategories() {
const categories = db.get('categories').value();
async function getCategories() {
const categories = await db_api.getRecords('categories');
return categories ? categories : null;
}
function getCategoriesAsPlaylists(files = null) {
async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = getCategories();
const available_categories = await getCategories();
if (available_categories && files) {
for (category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
@@ -97,10 +96,10 @@ function applyCategoryRules(file_json, rules, category_name) {
switch (rule['comparator']) {
case 'includes':
rule_applies = file_json[rule['property']].includes(rule['value']);
rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
break;
case 'not_includes':
rule_applies = !(file_json[rule['property']].includes(rule['value']));
rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
break;
case 'equals':
rule_applies = file_json[rule['property']] === rule['value'];

4
backend/config.js
View File

@@ -231,6 +231,10 @@ DEFAULT_CONFIG = {
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,

10
backend/consts.js
View File

@@ -153,6 +153,16 @@ let CONFIG_ITEMS = {
'path': 'YoutubeDLMaterial.Users.ldap_config'
},
// Database
'ytdl_use_local_db': {
'key': 'ytdl_use_local_db',
'path': 'YoutubeDLMaterial.Database.use_local_db'
},
'ytdl_mongodb_connection_string': {
'key': 'ytdl_mongodb_connection_string',
'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
},
// Advanced
'ytdl_default_downloader': {
'key': 'ytdl_default_downloader',

766
backend/db.js
View File

@@ -3,22 +3,102 @@ var path = require('path')
var utils = require('./utils')
const { uuid } = require('uuidv4');
const config_api = require('./config');
const { MongoClient } = require("mongodb");
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync');
const local_adapter = new FileSync('./appdata/local_db.json');
const local_db = low(local_adapter);
var logger = null;
var db = null;
var users_db = null;
function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db }
function setLogger(input_logger) { logger = input_logger; }
var database = null;
const tables = ['files', 'playlists', 'categories', 'subscriptions', 'downloads', 'users', 'roles', 'test'];
const local_db_defaults = {}
tables.forEach(table => {local_db_defaults[table] = []});
local_db.defaults(local_db_defaults).write();
let using_local_db = config_api.getConfigItem('ytdl_use_local_db');
function setDB(input_db, input_users_db) {
db = input_db; users_db = input_users_db;
exports.db = input_db;
exports.users_db = input_users_db
}
function setLogger(input_logger) {
logger = input_logger;
}
exports.initialize = (input_db, input_users_db, input_logger) => {
setDB(input_db, input_users_db);
setLogger(input_logger);
}
exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null) => {
exports.connectToDB = async (retries = 5, no_fallback = false) => {
if (using_local_db) return;
const success = await exports._connectToDB();
if (success) return true;
logger.warn(`MongoDB connection failed! Retrying ${retries} times...`);
const retry_delay_ms = 2000;
for (let i = 0; i < retries; i++) {
const retry_succeeded = await exports._connectToDB();
if (retry_succeeded) {
logger.info(`Successfully connected to DB after ${i+1} attempt(s)`);
return true;
}
if (i !== retries - 1) {
logger.warn(`Retry ${i+1} failed, waiting ${retry_delay_ms}ms before trying again.`);
await utils.wait(retry_delay_ms);
} else {
logger.warn(`Retry ${i+1} failed.`);
}
}
if (no_fallback) {
logger.error('Failed to connect to MongoDB. Verify your connection string is valid.');
return;
}
using_local_db = true;
config_api.setConfigItem('ytdl_use_local_db', true);
logger.error('Failed to connect to MongoDB, using Local DB as a fallback. Make sure your MongoDB instance is accessible, or set Local DB as a default through the config.');
return true;
}
exports._connectToDB = async () => {
const uri = config_api.getConfigItem('ytdl_mongodb_connection_string'); // "mongodb://127.0.0.1:27017/?compressors=zlib&gssapiServiceName=mongodb";
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
try {
await client.connect();
database = client.db('ytdl_material');
const existing_collections = (await database.listCollections({}, { nameOnly: true }).toArray()).map(collection => collection.name);
const missing_tables = tables.filter(table => !(existing_collections.includes(table)));
missing_tables.forEach(async table => {
await database.createCollection(table);
})
return true;
} catch(err) {
logger.error(err);
return false;
} finally {
// Ensures that the client will close when you finish/error
// await client.close();
}
}
exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => {
let db_path = null;
const file_id = utils.removeFileExtension(file_path);
const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
return false;
@@ -37,23 +117,9 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (!sub) {
if (multiUserMode) {
const user_uid = multiUserMode.user;
db_path = users_db.get('users').find({uid: user_uid}).get(`files`);
} else {
db_path = db.get(`files`);
}
} else {
if (multiUserMode) {
const user_uid = multiUserMode.user;
db_path = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
} else {
db_path = db.get('subscriptions').find({id: sub.id}).get('videos');
}
}
if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
const file_obj = registerFileDBManual(db_path, file_object);
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
@@ -63,18 +129,48 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
return file_obj;
}
function registerFileDBManual(db_path, file_object) {
exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject2(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms2(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile2(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
// remove duplicate(s)
db_path.remove({path: file_object['path']}).write();
exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
// add new file to db
db_path.push(file_object).write();
return file_object;
}
@@ -107,11 +203,38 @@ function generateFileObject(id, type, customPath = null, sub = null) {
return file_obj;
}
function generateFileObject2(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
var stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function getAppendedBasePathSub(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
exports.getFileDirectoriesAndDBs = () => {
exports.getFileDirectoriesAndDBs = async () => {
let dirs_to_check = [];
let subscriptions_to_check = [];
const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode
@@ -119,48 +242,45 @@ exports.getFileDirectoriesAndDBs = () => {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions');
if (multi_user_mode) {
let users = users_db.get('users').value();
const users = await exports.getRecords('users');
for (let i = 0; i < users.length; i++) {
const user = users[i];
if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']);
// add user's audio dir to check list
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'audio'),
dbPath: users_db.get('users').find({uid: user.uid}).get('files'),
user_uid: user.uid,
type: 'audio'
});
// add user's video dir to check list
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'video'),
dbPath: users_db.get('users').find({uid: user.uid}).get('files'),
type: 'video'
});
}
} else {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const subscriptions = db.get('subscriptions').value();
if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
// add audio dir to check list
dirs_to_check.push({
basePath: audioFolderPath,
dbPath: db.get('files'),
type: 'audio'
});
// add video dir to check list
dirs_to_check.push({
basePath: videoFolderPath,
dbPath: db.get('files'),
type: 'video'
});
}
if (subscriptions_enabled) {
const subscriptions = await exports.getRecords('subscriptions');
subscriptions_to_check = subscriptions_to_check.concat(subscriptions);
}
// add subscriptions to check list
for (let i = 0; i < subscriptions_to_check.length; i++) {
let subscription_to_check = subscriptions_to_check[i];
@@ -169,11 +289,11 @@ exports.getFileDirectoriesAndDBs = () => {
continue;
}
dirs_to_check.push({
basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
basePath: subscription_to_check.user_uid ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name)
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos')
: db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'),
type: subscription_to_check.type
user_uid: subscription_to_check.user_uid,
type: subscription_to_check.type,
sub_id: subscription_to_check['id']
});
}
@@ -181,22 +301,25 @@ exports.getFileDirectoriesAndDBs = () => {
}
exports.importUnregisteredFiles = async () => {
const dirs_to_check = exports.getFileDirectoriesAndDBs();
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (const dir_to_check of dirs_to_check) {
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => {
for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it
const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value())
const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id}))
if (!file_is_registered) {
// add additional info
registerFileDBManual(dir_to_check.dbPath, file);
await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
logger.verbose(`Added discovered file to the database: ${file.id}`);
}
});
}
}
}
@@ -204,26 +327,45 @@ exports.importUnregisteredFiles = async () => {
exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
const preimported_file_paths = [];
let dbPath = null;
if (sub.user_uid)
dbPath = users_db.get('users').find({uid: sub.user_uid}).get('subscriptions').find({id: sub.id}).get('videos');
else
dbPath = db.get('subscriptions').find({id: sub.id}).get('videos');
const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
files.forEach(file => {
for (let i = 0; i < files.length; i++) {
const file = files[i];
// check if file exists in db, if not add it
const file_is_registered = !!(dbPath.find({id: file.id}).value())
const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
if (!file_is_registered) {
// add additional info
registerFileDBManual(dbPath, file);
await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
preimported_file_paths.push(file['path']);
logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
}
});
}
return preimported_file_paths;
}
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await exports.bulkUpdateRecords('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => {
let new_playlist = {
name: playlist_name,
@@ -236,28 +378,19 @@ exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_u
const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid);
new_playlist.duration = duration;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
} else {
db.get(`playlists`)
.push(new_playlist)
.write();
}
new_playlist.user_uid = user_uid ? user_uid : undefined;
await exports.insertRecordIntoTable('playlists', new_playlist);
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = null
if (user_uid) {
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlist_id}).value();
} else {
playlist = db.get(`playlists`).find({id: playlist_id}).value();
}
let playlist = await exports.getRecord('playlists', {id: playlist_id});
if (!playlist) {
playlist = db.get('categories').find({uid: playlist_id}).value();
playlist = await exports.getRecord('categories', {uid: playlist_id});
if (playlist) {
// category found
const files = await exports.getFiles(user_uid);
@@ -271,7 +404,7 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i];
const uid = exports.getVideoUIDByID(fileName, user_uid);
const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
}
@@ -290,14 +423,21 @@ exports.updatePlaylist = async (playlist, user_uid = null) => {
const duration = await exports.calculatePlaylistDuration(playlist, user_uid);
playlist.duration = duration;
let db_loc = null;
if (user_uid) {
db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID});
} else {
db_loc = db.get(`playlists`).find({id: playlistID});
return await exports.updateRecord('playlists', {id: playlistID}, playlist);
}
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
}
db_loc.assign(playlist).write();
return true;
if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
}
exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => {
@@ -381,8 +521,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
base_db_path.get('files').remove({uid: uid}).write();
await exports.removeRecord('files', {uid: uid});
if (fileExists) {
await fs.unlink(file_obj.path);
@@ -398,29 +537,462 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = (file_id, uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
const file_obj = base_db_path.get('files').find({id: file_id}).value();
exports.getVideoUIDByID = async (file_id, uuid = null) => {
const file_obj = await exports.getRecord('files', {id: file_id});
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid, uuid = null, sub_id = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
return sub_db_path.find({uid: file_uid}).value();
return await exports.getRecord('files', {uid: file_uid});
}
exports.getFiles = async (uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
return base_db_path.get('files').value();
return await exports.getRecords('files', {user_uid: uuid});
}
exports.setVideoProperty = async (file_uid, assignment_obj, uuid, sub_id) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
const file_db_path = sub_db_path.find({uid: file_uid});
if (!(file_db_path.value())) {
logger.error(`Failed to find file with uid ${file_uid}`);
}
sub_db_path.find({uid: file_uid}).assign(assignment_obj).write();
exports.setVideoProperty = async (file_uid, assignment_obj) => {
// TODO: check if video exists, throw error if not
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
}
// DB to JSON
exports.exportDBToJSON = async (tables) => {
const users_db_json = await createUsersJSONs(tables.files, tables.playlists, tables.subscriptions, tables.categories, tables.users);
const db_json = await createNonUserJSON(tables.files, tables.playlists, tables.subscriptions, tables.categories);
return {users_db_json: users_db_json, db_json: db_json};
}
const createUsersJSONs = async (files, playlists, subscriptions, categories, users) => {
// we need to already have a list of user objects to gather the records into
for (let user of users) {
const files_of_user = files.filter(file => file.user_uid === user.uid && !file.sub_id);
const playlists_of_user = playlists.filter(playlist => playlist.user_uid === user.uid);
const subs_of_user = subscriptions.filter(sub => sub.user_uid === user.uid);
const categories_of_user = categories ? categories.filter(category => category && category.user_uid === user.uid) : [];
user['files'] = files_of_user;
user['playlists'] = playlists_of_user;
user['subscriptions'] = subs_of_user;
user['categories'] = categories_of_user;
for (let subscription of subscriptions) {
subscription['videos'] = files.filter(file => file.user_uid === user.uid && file.sub_id === sub.id);
}
}
}
const createNonUserJSON = async (files, playlists, subscriptions, categories) => {
const non_user_json = {
files: files.filter(file => !file.user_uid && !file.sub_id),
playlists: playlists.filter(playlist => !playlist.user_uid),
subscriptions: subscriptions.filter(sub => !sub.user_uid),
categories: categories ? categories.filter(category => category && !category.user_uid) : []
}
for (let subscription of non_user_json['subscriptions']) {
subscription['videos'] = files.filter(file => !file.user_uid && file.sub_id === subscription.id);
}
return non_user_json;
}
// Basic DB functions
// Create
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override
if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write();
return true;
}
if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
const output = await database.collection(table).insertOne(doc);
logger.debug(`Inserted doc into ${table}`);
return !!(output['result']['ok']);
}
exports.insertRecordsIntoTable = async (table, docs) => {
// local db override
if (using_local_db) {
local_db.get(table).push(...docs).write();
return true;
}
const output = await database.collection(table).insertMany(docs);
logger.debug(`Inserted ${output.insertedCount} docs into ${table}`);
return !!(output['result']['ok']);
}
exports.bulkInsertRecordsIntoTable = async (table, docs) => {
// local db override
if (using_local_db) {
return await exports.insertRecordsIntoTable(table, docs);
}
// not a necessary function as insertRecords does the same thing but gives us more control on batch size if needed
const table_collection = database.collection(table);
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
for (let i = 0; i < docs.length; i++) {
bulk.insert(docs[i]);
}
const output = await bulk.execute();
return !!(output['result']['ok']);
}
// Read
exports.getRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
}
return await database.collection(table).findOne(filter_obj);
}
exports.getRecords = async (table, filter_obj = null) => {
// local db override
if (using_local_db) {
return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
}
return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray();
}
// Update
exports.updateRecord = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
return true;
}
// sometimes _id will be in the update obj, this breaks mongodb
if (update_obj['_id']) delete update_obj['_id'];
const output = await database.collection(table).updateOne(filter_obj, {$set: update_obj});
return !!(output['result']['ok']);
}
exports.updateRecords = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
return true;
}
const output = await database.collection(table).updateMany(filter_obj, {$set: update_obj});
return !!(output['result']['ok']);
}
exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
// local db override
if (using_local_db) {
local_db.get(table).each((record) => {
const item_id_to_update = record[key_label];
if (!update_obj[item_id_to_update]) return;
const props_to_update = Object.keys(update_obj[item_id_to_update]);
for (let i = 0; i < props_to_update.length; i++) {
const prop_to_update = props_to_update[i];
const prop_value = update_obj[item_id_to_update][prop_to_update];
record[prop_to_update] = prop_value;
}
}).write();
return true;
}
const table_collection = database.collection(table);
let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
const item_ids_to_update = Object.keys(update_obj);
for (let i = 0; i < item_ids_to_update.length; i++) {
const item_id_to_update = item_ids_to_update[i];
bulk.find({[key_label]: item_id_to_update }).updateOne({
"$set": update_obj[item_id_to_update]
});
}
const output = await bulk.execute();
return !!(output['result']['ok']);
}
exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$push: {[key]: value}});
return !!(output['result']['ok']);
}
exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$pull: {[key]: value}});
return !!(output['result']['ok']);
}
// Delete
exports.removeRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
return true;
}
const output = await database.collection(table).deleteOne(filter_obj);
return !!(output['result']['ok']);
}
exports.removeAllRecords = async (table = null) => {
// local db override
if (using_local_db) {
const tables_to_remove = table ? [table] : tables;
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
local_db.assign({[table_to_remove]: []}).write();
logger.debug(`Removed all records from ${table_to_remove}`);
}
return true;
}
let success = true;
const tables_to_remove = table ? [table] : tables;
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
const output = await database.collection(table_to_remove).deleteMany({});
logger.debug(`Removed all records from ${table_to_remove}`);
success &= !!(output['result']['ok']);
}
return success;
}
// Stats
exports.getDBStats = async () => {
const stats_by_table = {};
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
if (table === 'test') continue;
stats_by_table[table] = await getDBTableStats(table);
}
return {stats_by_table: stats_by_table, using_local_db: using_local_db};
}
const getDBTableStats = async (table) => {
const table_stats = {};
// local db override
if (using_local_db) {
table_stats['records_count'] = local_db.get(table).value().length;
} else {
const stats = await database.collection(table).stats();
table_stats['records_count'] = stats.count;
}
return table_stats;
}
// JSON to DB
exports.generateJSONTables = async (db_json, users_json) => {
// create records
let files = db_json['files'] || [];
let playlists = db_json['playlists'] || [];
let categories = db_json['categories'] || [];
let subscriptions = db_json['subscriptions'] || [];
const users = users_json['users'];
for (let i = 0; i < users.length; i++) {
const user = users[i];
if (user['files']) {
user['files'] = user['files'].map(file => ({ ...file, user_uid: user['uid'] }));
files = files.concat(user['files']);
}
if (user['playlists']) {
user['playlists'] = user['playlists'].map(playlist => ({ ...playlist, user_uid: user['uid'] }));
playlists = playlists.concat(user['playlists']);
}
if (user['categories']) {
user['categories'] = user['categories'].map(category => ({ ...category, user_uid: user['uid'] }));
categories = categories.concat(user['categories']);
}
if (user['subscriptions']) {
user['subscriptions'] = user['subscriptions'].map(subscription => ({ ...subscription, user_uid: user['uid'] }));
subscriptions = subscriptions.concat(user['subscriptions']);
}
}
const tables_obj = {};
// TODO: use create*Records funcs to strip unnecessary properties
tables_obj.files = createFilesRecords(files, subscriptions);
tables_obj.playlists = playlists;
tables_obj.categories = categories;
tables_obj.subscriptions = createSubscriptionsRecords(subscriptions);
tables_obj.users = createUsersRecords(users);
tables_obj.roles = createRolesRecords(users_json['roles']);
tables_obj.downloads = createDownloadsRecords(db_json['downloads'])
return tables_obj;
}
exports.importJSONToDB = async (db_json, users_json) => {
// TODO: backup db
// TODO: delete current records
const tables_obj = await exports.generateJSONTables(db_json, users_json);
const table_keys = Object.keys(tables_obj);
let success = true;
for (let i = 0; i < table_keys.length; i++) {
const table_key = table_keys[i];
success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key]);
}
return success;
}
const createFilesRecords = (files, subscriptions) => {
for (let i = 0; i < subscriptions.length; i++) {
const subscription = subscriptions[i];
subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
files = files.concat(subscriptions[i]['videos']);
console.log(files.length);
}
return files;
}
const createPlaylistsRecords = async (playlists) => {
}
const createCategoriesRecords = async (categories) => {
}
const createSubscriptionsRecords = (subscriptions) => {
for (let i = 0; i < subscriptions.length; i++) {
delete subscriptions[i]['videos'];
}
return subscriptions;
}
const createUsersRecords = (users) => {
users.forEach(user => {
delete user['files'];
delete user['playlists'];
delete user['subscriptions'];
});
return users;
}
const createRolesRecords = (roles) => {
const new_roles = [];
Object.keys(roles).forEach(role_key => {
new_roles.push({
key: role_key,
...roles[role_key]
});
});
return new_roles;
}
const createDownloadsRecords = (downloads) => {
const new_downloads = [];
Object.keys(downloads).forEach(session_key => {
new_downloads.push({
key: session_key,
...downloads[session_key]
});
});
return new_downloads;
}
exports.transferDB = async (local_to_remote) => {
const table_to_records = {};
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
table_to_records[table] = await exports.getRecords(table);
}
using_local_db = !local_to_remote;
if (local_to_remote) {
// backup local DB
logger.debug('Backup up Local DB...');
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
const db_connected = await exports.connectToDB(5, true);
if (!db_connected) {
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
return false;
}
}
success = true;
logger.debug('Clearing new database before transfer...');
await exports.removeAllRecords();
logger.debug('Database cleared! Beginning transfer.');
for (let i = 0; i < tables.length; i++) {
const table = tables[i];
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
}
config_api.setConfigItem('ytdl_use_local_db', using_local_db);
return success;
}
/*
This function is necessary to emulate mongodb's ability to search for null or missing values.
A filter of null or undefined for a property will find docs that have that property missing, or have it
null or undefined. We want that same functionality for the local DB as well
*/
const applyFilterLocalDB = (db_path, filter_obj, operation) => {
const filter_props = Object.keys(filter_obj);
const return_val = db_path[operation](record => {
if (!filter_props) return true;
let filtered = true;
for (let i = 0; i < filter_props.length; i++) {
const filter_prop = filter_props[i];
const filter_prop_value = filter_obj[filter_prop];
if (filter_prop_value === undefined || filter_prop_value === null) {
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
} else {
filtered &= record[filter_prop] === filter_prop_value;
}
}
return filtered;
});
return return_val;
}

77
backend/package-lock.json generated
View File

@@ -441,6 +441,11 @@
"resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz",
"integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw=="
},
"bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg=="
},
"buffer": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
@@ -923,6 +928,11 @@
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"denque": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -2042,6 +2052,12 @@
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
},
"memory-pager": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
"optional": true
},
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
@@ -2196,6 +2212,44 @@
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
},
"mongodb": {
"version": "3.6.9",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.9.tgz",
"integrity": "sha512-1nSCKgSunzn/CXwgOWgbPHUWOO5OfERcuOWISmqd610jn0s8BU9K4879iJVabqgpPPbA6hO7rG48eq+fGED3Mg==",
"requires": {
"bl": "^2.2.1",
"bson": "^1.1.4",
"denque": "^1.4.1",
"optional-require": "^1.0.3",
"safe-buffer": "^5.1.2",
"saslprep": "^1.0.0"
},
"dependencies": {
"bl": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
}
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -2423,6 +2477,11 @@
"mimic-fn": "^2.1.0"
}
},
"optional-require": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.3.tgz",
"integrity": "sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA=="
},
"p-cancelable": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz",
@@ -2820,6 +2879,15 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
"optional": true,
"requires": {
"sparse-bitfield": "^3.0.3"
}
},
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
@@ -2930,6 +2998,15 @@
"is-arrayish": "^0.3.1"
}
},
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
"optional": true,
"requires": {
"memory-pager": "^1.0.2"
}
},
"sshpk": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",

4
backend/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon app.js"
"start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
},
"nodemonConfig": {
"ignore": [
@@ -46,6 +47,7 @@
"merge-files": "^0.1.2",
"mocha": "^8.4.0",
"moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"node-id3": "^0.1.14",

164
backend/subscriptions.js
View File

@@ -14,13 +14,13 @@ const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
let db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setDB(input_db_api) { db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
function initialize(input_db_api, input_logger) {
setDB(input_db_api);
setLogger(input_logger);
}
@@ -34,12 +34,7 @@ async function subscribe(sub, user_uid = null) {
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = false;
if (user_uid)
url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
else
url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
if (!sub.name && url_exists) {
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -48,19 +43,12 @@ async function subscribe(sub, user_uid = null) {
return;
}
// add sub to db
let sub_db = null;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub, user_uid);
if (success) {
sub = sub_db.value();
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
@@ -91,8 +79,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, function(err, output) {
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id);
}
@@ -122,10 +110,7 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
// if it's now valid, update
if (sub.name) {
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
}
}
@@ -141,10 +126,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
// updates subscription
sub.archive = archive_dir;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
}
// TODO: get even more info
@@ -166,10 +149,8 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
let result_obj = { success: false, error: '' };
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
await db_api.removeRecord('subscriptions', {id: id});
await db_api.removeAllRecords('files', {sub_id: id});
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
@@ -191,20 +172,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
// TODO: combine this with deletefile
let basePath = null;
let sub_db = null;
if (user_uid) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
sub_db = db.get('subscriptions').find({id: sub.id});
}
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
await db_api.removeRecord('files', {uid: file_uid});
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
@@ -255,14 +232,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
}
async function getVideosForSub(sub, user_uid = null) {
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
const latest_sub_obj = sub_db.value();
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -292,12 +262,12 @@ async function getVideosForSub(sub, user_uid = null) {
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
return new Promise(async resolve => {
const preimported_file_paths = [];
const PREIMPORT_INTERVAL = 5000;
const preregister_check = setInterval(() => {
const preregister_check = setInterval(async () => {
if (sub.streamingOnly) return;
db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
}, PREIMPORT_INTERVAL);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
@@ -313,7 +283,7 @@ async function getVideosForSub(sub, user_uid = null) {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
await handleOutputJSON(sub, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
@@ -347,7 +317,7 @@ async function getVideosForSub(sub, user_uid = null) {
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, preimported_file_paths, reset_videos);
await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
}
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
@@ -444,8 +414,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
return downloadConfig;
}
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) {
async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
// TODO: remove streaming only mode
if (false && sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
@@ -459,12 +430,15 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object);
if (sub_db.get('videos').find({path: path_string}).value()) {
const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id});
if (file_exists) {
// TODO: fix issue where files of different paths due to custom path get downloaded multiple times
// file already exists in DB, return early to avoid reseting the download date
return;
}
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);
const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
@@ -477,73 +451,41 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
}
}
function getSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
else
return db.get('subscriptions').value();
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
function getAllSubscriptions() {
let subscriptions = null;
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (multiUserMode) {
subscriptions = [];
let users = users_db.get('users').value();
for (let i = 0; i < users.length; i++) {
if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']);
}
} else {
subscriptions = getSubscriptions();
}
return subscriptions;
return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
}
function getSubscription(subID, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return db.get('subscriptions').find({id: subID}).value();
async function getSubscription(subID) {
return await db_api.getRecord('subscriptions', {id: subID});
}
function getSubscriptionByName(subName, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value();
else
return db.get('subscriptions').find({name: subName}).value();
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
function updateSubscription(sub, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(sub).write();
}
async function updateSubscription(sub, user_uid = null) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
return true;
}
function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(sub => {
updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
});
}
function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
}
async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
// TODO: combine with updateSubscription
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
return true;
}
function subExists(subID, user_uid = null) {
if (user_uid)
return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return !!db.get('subscriptions').find({id: subID}).value();
}
async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
@@ -559,7 +501,7 @@ async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
checkVideoIfBetterExists(video, sub, user_uid)
await checkVideoIfBetterExists(video, sub, user_uid)
}
});
}
@@ -569,7 +511,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
if (err) {
// video is not available anymore for whatever reason
} else if (output) {

165
backend/test/tests.js
View File

@@ -29,7 +29,7 @@ const logger = winston.createLogger({
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
new winston.transports.Console({level: 'debug', name: 'console'})
]
});
@@ -38,19 +38,167 @@ var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db, logger);
auth_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api);
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(true);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Transfer to local', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(false);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
});
describe('Export', function() {
});
describe('Import and Export', async function() {
it('Existing data', async function() {
const users_db_json = users_db.value();
const db_json = db.value();
const users_db_json_stringified = JSON.stringify(users_db_json);
const db_json_stringified = JSON.stringify(db_json);
const tables_obj = await db_api.importJSONtoDB(users_db_json, db_json);
const db_jsons = await db_api.exportDBToJSON(tables_obj);
const users_db_json_returned_stringified = db_jsons['users_db_json'];
const db_json_returned_stringified = db_jsons['db_json'];
assert(users_db_json_returned_stringified.length === users_db_json_stringified.length);
assert(db_json_returned_stringified.length === db_json_stringified.length);
});
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
});
});
describe('Multi User', async function() {
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
before(async function() {
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
console.log('hi')
});
describe('Authentication', function() {
it('login', async function() {
@@ -93,11 +241,12 @@ describe('Multi User', async function() {
});
it('Subscription zip generator', async function() {
const sub = subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
assert(sub);
const sub_files_to_download = [];
for (let i = 0; i < sub['videos'].length; i++) {
const sub_file = sub['videos'][i];
for (let i = 0; i < sub_videos.length; i++) {
const sub_file = sub_videos[i];
sub_files_to_download.push(sub_file);
}
const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);

70
backend/utils.js
View File

@@ -122,6 +122,21 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj;
}
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
}
else obj = 0;
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
@@ -143,6 +158,23 @@ function getDownloadedThumbnail(name, type, customPath = null) {
return null;
}
function getDownloadedThumbnail2(file_path, type) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
let pngPath = file_path_no_extension + '.png';
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getExpectedFileSize(input_info_jsons) {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
@@ -190,6 +222,28 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
}
}
function fixVideoMetadataPerms2(file_path, type) {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
file_path_no_extension + '.info.json',
file_path_no_extension + ext + '.info.json',
// Thumbnails
file_path_no_extension + '.webp',
file_path_no_extension + '.jpg'
];
for (const file of files_to_fix) {
if (!fs.existsSync(file)) continue;
fs.chmodSync(file, 0o644);
}
}
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
@@ -202,6 +256,18 @@ function deleteJSONFile(name, type, customPath = null) {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
function deleteJSONFile2(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
@@ -309,11 +375,15 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getDownloadedThumbnail2: getDownloadedThumbnail2,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
fixVideoMetadataPerms2: fixVideoMetadataPerms2,
deleteJSONFile: deleteJSONFile,
deleteJSONFile2: deleteJSONFile2,
removeIDFromArchive, removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,

16
src/app/components/downloads/downloads.component.html
View File

@@ -1,21 +1,21 @@
<div style="padding: 20px;">
<div *ngFor="let session_downloads of downloads | keyvalue">
<ng-container *ngIf="keys(session_downloads.value).length > 0">
<div *ngFor="let session_downloads of downloads">
<ng-container *ngIf="keys(session_downloads).length > 2">
<mat-card style="padding-bottom: 30px; margin-bottom: 15px;">
<h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads.key}}
<span *ngIf="session_downloads.key === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
<h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads['session_id']}}
<span *ngIf="session_downloads['session_id'] === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
</h4>
<div class="container">
<div class="row">
<div *ngFor="let download of session_downloads.value | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
<mat-card *ngIf="download.value" class="mat-elevation-z3">
<app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads.key, download.value.uid)"></app-download-item>
<div *ngFor="let download of session_downloads | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
<mat-card *ngIf="download.key !== 'session_id' && download.key !== '_id' && download.value" class="mat-elevation-z3">
<app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads['session_id'], download.value.uid)"></app-download-item>
</mat-card>
</div>
</div>
</div>
<div>
<button style="top: 15px;" (click)="clearDownloads(session_downloads.key)" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
<button style="top: 15px;" (click)="clearDownloads(session_downloads['session_id'])" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
</div>
</mat-card>
</ng-container>

12
src/app/components/downloads/downloads.component.ts
View File

@@ -35,7 +35,7 @@ import { Router } from '@angular/router';
export class DownloadsComponent implements OnInit, OnDestroy {
downloads_check_interval = 1000;
downloads = {};
downloads = [];
interval_id = null;
keys = Object.keys;
@@ -137,6 +137,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
this.downloads[session_id] = session_downloads_by_id;
} else {
for (let j = 0; j < session_download_ids.length; j++) {
if (session_download_ids[j] === 'session_id' || session_download_ids[j] === '_id') continue;
const download_id = session_download_ids[j];
const download = new_downloads_by_session[session_id][download_id]
if (!this.downloads[session_id][download_id]) {
@@ -156,11 +157,10 @@ export class DownloadsComponent implements OnInit, OnDestroy {
downloadsValid() {
let valid = false;
const keys = this.keys(this.downloads);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const value = this.downloads[key];
if (this.keys(value).length > 0) {
for (let i = 0; i < this.downloads.length; i++) {
const session_downloads = this.downloads[i];
if (!session_downloads) continue;
if (this.keys(session_downloads).length > 2) {
valid = true;
break;
}

2
src/app/components/manage-role/manage-role.component.html
View File

@@ -5,7 +5,7 @@
<mat-list-item role="listitem" *ngFor="let permission of available_permissions">
<h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
<span matLine>
<mat-radio-group [disabled]="permission === 'settings' && role.name === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
<mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
<mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
<mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
</mat-radio-group>

2
src/app/components/manage-role/manage-role.component.ts
View File

@@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit {
}
changeRolePermissions(change, permission) {
this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => {
this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => {
if (res['success']) {
} else {

2
src/app/components/modify-users/modify-users.component.html
View File

@@ -94,7 +94,7 @@
</div>
<button color="primary" [matMenuTriggerFor]="edit_roles_menu" class="edit-role" mat-raised-button><ng-container i18n="Edit role">Edit Role</ng-container></button>
<mat-menu #edit_roles_menu="matMenu">
<button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.name}}</button>
<button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.key}}</button>
</mat-menu>
</div>

11
src/app/components/modify-users/modify-users.component.ts
View File

@@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
getRoles() {
this.postsService.getRoles().subscribe(res => {
this.roles = [];
const roles = res['roles'];
const role_names = Object.keys(roles);
for (let i = 0; i < role_names.length; i++) {
const role_name = role_names[i];
this.roles.push({
name: role_name,
permissions: roles[role_name]['permissions']
});
}
this.roles = res['roles'];
});
}

2
src/app/dialogs/modify-playlist/modify-playlist.component.ts
View File

@@ -61,7 +61,7 @@ export class ModifyPlaylistComponent implements OnInit {
}
playlistChanged() {
return JSON.stringify(this.playlist) === JSON.stringify(this.original_playlist);
return JSON.stringify(this.playlist) !== JSON.stringify(this.original_playlist);
}
getPlaylist() {

13
src/app/main/main.component.html
View File

@@ -20,11 +20,16 @@
</ng-container>
</mat-label>
<mat-select [ngModelOptions]="{standalone: true}" [(ngModel)]="selectedQuality">
<ng-container *ngFor="let option of qualityOptions[(audioOnly) ? 'audio' : 'video']">
<mat-option *ngIf="option.value === '' || url && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats'] && cachedAvailableFormats[url]['formats'][(audioOnly) ? 'audio' : 'video'][option.value]" [value]="option.value">
{{option.label}}
<mat-option [value]="''">
Max
</mat-option>
</ng-container>
<ng-container *ngIf="url && cachedAvailableFormats && cachedAvailableFormats[url]?.formats">
<ng-container *ngFor="let option of cachedAvailableFormats[url]['formats'][audioOnly ? 'audio' : 'video']">
<mat-option *ngIf="option.key !== 'best_audio_format'" [value]="option">
{{option.key}}
</mat-option>
</ng-container>
</ng-container>
</mat-select>
<div class="spinner-div" *ngIf="url !== '' && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats_loading']">
<mat-spinner [diameter]="25"></mat-spinner>

56
src/app/main/main.component.ts
View File

@@ -500,23 +500,26 @@ export class MainComponent implements OnInit {
}
getSelectedAudioFormat() {
if (this.selectedQuality === '') { return null };
if (this.selectedQuality === '') { return null; }
const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
if (cachedFormatsExists) {
const audio_formats = this.cachedAvailableFormats[this.url]['formats']['audio'];
return audio_formats[this.selectedQuality]['format_id'];
return this.selectedQuality['format_id'];
} else {
return null;
}
}
getSelectedVideoFormat() {
if (this.selectedQuality === '') { return null };
const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
if (cachedFormatsExists) {
const video_formats = this.cachedAvailableFormats[this.url]['formats']['video'];
if (video_formats['best_audio_format'] && this.selectedQuality !== '') {
return video_formats[this.selectedQuality]['format_id'] + '+' + video_formats['best_audio_format'];
if (this.selectedQuality === '') { return null; }
const cachedFormats = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
if (cachedFormats) {
const video_formats = cachedFormats['video'];
if (this.selectedQuality) {
let selected_video_format = this.selectedQuality['format_id'];
// add in audio format if necessary
if (!this.selectedQuality['acodec'] && cachedFormats['best_audio_format']) selected_video_format += `+${cachedFormats['best_audio_format']}`;
return selected_video_format;
}
}
return null;
@@ -644,9 +647,8 @@ export class MainComponent implements OnInit {
this.errorFormats(url);
return;
}
const parsed_infos = this.getAudioAndVideoFormats(infos.formats);
const available_formats = {audio: parsed_infos[0], video: parsed_infos[1]};
this.cachedAvailableFormats[url]['formats'] = available_formats;
this.cachedAvailableFormats[url]['formats'] = this.getAudioAndVideoFormats(infos.formats);
console.log(this.cachedAvailableFormats[url]['formats']);
}, err => {
this.errorFormats(url);
});
@@ -689,7 +691,7 @@ export class MainComponent implements OnInit {
if (audio_format) {
format_array.push('-f', audio_format);
} else if (this.selectedQuality) {
format_array.push('--audio-quality', this.selectedQuality);
format_array.push('--audio-quality', this.selectedQuality['format_id']);
}
// pushes formats
@@ -705,7 +707,7 @@ export class MainComponent implements OnInit {
if (video_format) {
format_array = ['-f', video_format];
} else if (this.selectedQuality) {
format_array = [`bestvideo[height=${this.selectedQuality}]+bestaudio/best[height=${this.selectedQuality}]`];
format_array = [`bestvideo[height=${this.selectedQuality['format_id']}]+bestaudio/best[height=${this.selectedQuality}]`];
}
// pushes formats
@@ -802,9 +804,11 @@ export class MainComponent implements OnInit {
}
}
getAudioAndVideoFormats(formats): any[] {
const audio_formats = {};
const video_formats = {};
getAudioAndVideoFormats(formats) {
const audio_formats: any = {};
const video_formats: any = {};
console.log(formats);
for (let i = 0; i < formats.length; i++) {
const format_obj = {type: null};
@@ -815,9 +819,12 @@ export class MainComponent implements OnInit {
format_obj.type = format_type;
if (format_obj.type === 'audio' && format.abr) {
const key = format.abr.toString() + 'K';
format_obj['key'] = key;
format_obj['bitrate'] = format.abr;
format_obj['format_id'] = format.format_id;
format_obj['ext'] = format.ext;
format_obj['label'] = key;
// don't overwrite if not m4a
if (audio_formats[key]) {
if (format.ext === 'm4a') {
@@ -828,11 +835,14 @@ export class MainComponent implements OnInit {
}
} else if (format_obj.type === 'video') {
// check if video format is mp4
const key = format.format_note.replace('p', '');
const key = `${format.height}p${Math.round(format.fps)}`;
if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') {
format_obj['key'] = key;
format_obj['height'] = format.height;
format_obj['acodec'] = format.acodec;
format_obj['format_id'] = format.format_id;
format_obj['label'] = key;
format_obj['fps'] = Math.round(format.fps);
// no acodec means no overwrite
if (!(video_formats[key]) || format_obj['acodec'] !== 'none') {
@@ -842,9 +852,17 @@ export class MainComponent implements OnInit {
}
}
video_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats);
const parsed_formats: any = {};
return [audio_formats, video_formats]
parsed_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats);
parsed_formats['video'] = Object.values(video_formats);
parsed_formats['audio'] = Object.values(audio_formats);
parsed_formats['video'] = parsed_formats['video'].sort((a, b) => b.height - a.height || b.fps - a.fps);
parsed_formats['audio'] = parsed_formats['audio'].sort((a, b) => b.bitrate - a.bitrate);
return parsed_formats;
}
getBestAudioFormatForMp4(audio_formats) {

4
src/app/player/player.component.html
View File

@@ -27,11 +27,11 @@
</ng-container>
</div>
<div class="col-2">
<ng-container *ngIf="playlist.length > 1">
<ng-container *ngIf="db_playlist">
<button (click)="downloadContent()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
</ng-container>
<ng-container *ngIf="playlist.length === 1">
<ng-container *ngIf="db_file">
<button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
<button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button>

9
src/app/player/player.component.ts
View File

@@ -171,7 +171,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
console.error('Failed to increment view count');
console.error(err);
});
this.uids = this.db_file['uid'];
this.uids = [this.db_file['uid']];
this.show_player = true;
this.parseFileNames();
}
@@ -304,12 +304,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
}
downloadContent() {
const fileNames = [];
for (let i = 0; i < this.playlist.length; i++) {
fileNames.push(this.playlist[i].title);
}
const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0];
const zipName = this.db_playlist.name;
this.downloading = true;
this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => {
this.downloading = false;

17
src/app/posts.services.ts
View File

@@ -184,6 +184,18 @@ export class PostsService implements CanActivate {
cropFileSettings: cropFileSettings}, this.httpOptions);
}
getDBInfo() {
return this.http.post(this.path + 'getDBInfo', {}, this.httpOptions);
}
transferDB(local_to_remote) {
return this.http.post(this.path + 'transferDB', {local_to_remote: local_to_remote}, this.httpOptions);
}
testConnectionString() {
return this.http.post(this.path + 'testConnectionString', {}, this.httpOptions);
}
killAllDownloads() {
return this.http.post(this.path + 'killAllDownloads', {}, this.httpOptions);
}
@@ -236,11 +248,12 @@ export class PostsService implements CanActivate {
return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions);
}
downloadFileFromServer(uid, uuid = null, sub_id = null) {
downloadFileFromServer(uid, uuid = null, sub_id = null, is_playlist = null) {
return this.http.post(this.path + 'downloadFileFromServer', {
uid: uid,
uuid: uuid,
sub_id: sub_id
sub_id: sub_id,
is_playlist: is_playlist
},
{responseType: 'blob', params: this.httpOptions.params});
}

37
src/app/settings/settings.component.html
View File

@@ -280,6 +280,43 @@
</div>
</ng-template>
</mat-tab>
<!-- Database -->
<mat-tab label="Database" i18n-label="Database settings label">
<ng-template matTabContent>
<div *ngIf="new_config" class="container-fluid">
<div class="row">
<div class="col-12 mt-3">
<div *ngIf="db_info">
<h5 i18n="Database info title">Database Info</h5>
<p><ng-container i18n="Database location label">Database location:</ng-container>&nbsp;<strong>{{db_info['using_local_db'] ? 'Local' : 'MongoDB'}}</strong></p>
<h6 i18n="Records per table label">Records per table</h6>
<mat-list style="padding-top: 0px">
<mat-list-item style="height: 28px" *ngFor="let table_stats of db_info['stats_by_table'] | keyvalue">
{{table_stats.key}}: {{table_stats.value.records_count}}
</mat-list-item>
</mat-list>
<mat-form-field style="width: 100%; margin-top: 15px; margin-bottom: 10px" color="accent">
<input [(ngModel)]="new_config['Database']['mongodb_connection_string']" matInput placeholder="MongoDB Connection String" i18n-placeholder="MongoDB Connection String" required>
<mat-hint><ng-container i18n="MongoDB Connection String setting hint AKA preamble">Example:</ng-container>&nbsp;mongodb://127.0.0.1:27017/?compressors=zlib</mat-hint>
</mat-form-field>
<br>
<button (click)="testConnectionString()" [disabled]="testing_connection_string" mat-flat-button color="accent"><ng-container i18n="Test connection string button">Test connection string</ng-container></button>
<br>
<button class="transfer-db-button" [disabled]="db_transferring" color="accent" (click)="transferDB()" mat-raised-button><ng-container i18n="Transfer DB button">Transfer DB to </ng-container>{{db_info['using_local_db'] ? 'MongoDB' : 'Local'}}</button>
</div>
<div *ngIf="!db_info">
<ng-container i18n="Database info not retrieved error message">Database information could not be retrieved. Check the server logs for more information.</ng-container>
</div>
</div>
</div>
</div>
</ng-template>
</mat-tab>
<!-- Advanced -->
<mat-tab label="Advanced" i18n-label="Host settings label">
<ng-template matTabContent>

13
src/app/settings/settings.component.scss
View File

@@ -77,8 +77,13 @@
}
.category-custom-placeholder {
background: #ccc;
border: dotted 3px #999;
min-height: 60px;
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
background: #ccc;
border: dotted 3px #999;
min-height: 60px;
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
}
.transfer-db-button {
margin-top: 10px;
margin-bottom: 10px;
}

59
src/app/settings/settings.component.ts
View File

@@ -29,6 +29,10 @@ export class SettingsComponent implements OnInit {
generated_bookmarklet_code = null;
bookmarkletAudioOnly = false;
db_info = null;
db_transferring = false;
testing_connection_string = false;
_settingsSame = true;
latestGithubRelease = null;
@@ -48,6 +52,7 @@ export class SettingsComponent implements OnInit {
ngOnInit() {
this.getConfig();
this.getDBInfo();
this.generated_bookmarklet_code = this.sanitizer.bypassSecurityTrustUrl(this.generateBookmarkletCode());
@@ -263,6 +268,60 @@ export class SettingsComponent implements OnInit {
});
}
getDBInfo() {
this.postsService.getDBInfo().subscribe(res => {
this.db_info = res['db_info'];
});
}
transferDB() {
const dialogRef = this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: 'Transfer DB',
dialogText: `Are you sure you want to transfer the DB?`,
submitText: 'Transfer',
}
});
dialogRef.afterClosed().subscribe(confirmed => {
if (confirmed) {
this._transferDB();
}
});
}
_transferDB() {
this.db_transferring = true;
this.postsService.transferDB(this.db_info['using_local_db']).subscribe(res => {
this.db_transferring = false;
const success = res['success'];
if (success) {
this.openSnackBar('Successfully transfered DB! Reloading info...');
this.getDBInfo();
} else {
this.openSnackBar('Failed to transfer DB -- transfer was aborted. Error: ' + res['error']);
}
}, err => {
this.db_transferring = false;
this.openSnackBar('Failed to transfer DB -- API call failed. See browser logs for details.');
console.error(err);
});
}
testConnectionString() {
this.testing_connection_string = true;
this.postsService.testConnectionString().subscribe(res => {
this.testing_connection_string = false;
if (res['success']) {
this.postsService.openSnackBar('Connection successful!');
} else {
this.postsService.openSnackBar('Connection failed! Error: ' + res['error']);
}
}, err => {
this.testing_connection_string = false;
this.postsService.openSnackBar('Connection failed! Error: Server error. See logs for more info.');
});
}
// snackbar helper
public openSnackBar(message: string, action: string = '') {
this.snackBar.open(message, action, {