Merge pull request #655 from Tzahi12345/improved-downloads-management

Improved downloads management
This commit is contained in:
Tzahi12345
2022-06-20 16:07:02 -04:00
committed by GitHub
149 changed files with 939 additions and 736 deletions

View File

@@ -933,23 +933,34 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
let file_count = await db_api.getRecords('files', filter_obj, true);
playlists = await db_api.getRecords('playlists', {user_uid: uuid});
const categories = await categories_api.getCategoriesAsPlaylists(files);
if (categories) {
playlists = playlists.concat(categories);
}
const file_count = await db_api.getRecords('files', filter_obj, true);
files = JSON.parse(JSON.stringify(files));
res.send({
files: files,
file_count: file_count,
playlists: playlists
});
});
app.post('/api/updateFile', optionalJwt, async function (req, res) {
const uid = req.body.uid;
const change_obj = req.body.change_obj;
const file = await db_api.updateRecord('files', {uid: uid}, change_obj);
if (!file) {
res.send({
success: false,
error: 'File could not be found'
});
} else {
res.send({
success: true
});
}
});
app.post('/api/checkConcurrentStream', async (req, res) => {
const uid = req.body.uid;
@@ -1365,7 +1376,7 @@ app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
let playlists = await db_api.getRecords('playlists', {user_uid: uuid});
if (include_categories) {
const categories = await categories_api.getCategoriesAsPlaylists(files);
const categories = await categories_api.getCategoriesAsPlaylists();
if (categories) {
playlists = playlists.concat(categories);
}
@@ -1669,9 +1680,15 @@ app.post('/api/download', optionalJwt, async (req, res) => {
}
});
app.post('/api/clearFinishedDownloads', optionalJwt, async (req, res) => {
app.post('/api/clearDownloads', optionalJwt, async (req, res) => {
const user_uid = req.isAuthenticated() ? req.user.uid : null;
const success = db_api.removeAllRecords('download_queue', {finished: true, user_uid: user_uid});
const clear_finished = req.body.clear_finished;
const clear_paused = req.body.clear_paused;
const clear_errors = req.body.clear_errors;
let success = true;
if (clear_finished) success &= await db_api.removeAllRecords('download_queue', {finished: true, user_uid: user_uid});
if (clear_paused) success &= await db_api.removeAllRecords('download_queue', {paused: true, user_uid: user_uid});
if (clear_errors) success &= await db_api.removeAllRecords('download_queue', {error: {$ne: null}, user_uid: user_uid});
res.send({success: success});
});

View File

@@ -171,8 +171,12 @@ exports.registerUser = async function(req, res) {
exports.login = async (username, password) => {
// even if we're using LDAP, we still want users to be able to login using internal credentials
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); return false }
if (!user) {
if (config_api.getConfigItem('ytdl_auth_method') === 'internal') logger.error(`User ${username} not found`);
return false;
}
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
}

View File

@@ -55,17 +55,18 @@ async function getCategories() {
return categories ? categories : null;
}
async function getCategoriesAsPlaylists(files = null) {
async function getCategoriesAsPlaylists() {
const categories_as_playlists = [];
const available_categories = await getCategories();
if (available_categories && files) {
if (available_categories) {
for (let category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
const files_that_match = await db_api.getRecords('files', {'category.uid': category['uid']});
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
category['thumbnailPath'] = files_that_match[0].thumbnailPath;
category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
category['id'] = category['uid'];
category['auto'] = true;
categories_as_playlists.push(category);
}
}

View File

@@ -127,7 +127,7 @@ function setConfigItem(key, value) {
success = setConfigFile(config_json);
return success;
};
}
function setConfigItems(items) {
let success = false;

View File

@@ -387,9 +387,9 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
if (!playlist) {
playlist = await exports.getRecord('categories', {uid: playlist_id});
if (playlist) {
// category found
const files = await exports.getFiles(user_uid);
utils.addUIDsToCategory(playlist, files);
const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
playlist['uids'] = uids;
playlist['auto'] = true;
}
}
@@ -629,7 +629,7 @@ exports.bulkInsertRecordsIntoTable = async (table, docs) => {
exports.getRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
return exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
}
return await database.collection(table).findOne(filter_obj);
@@ -638,7 +638,7 @@ exports.getRecord = async (table, filter_obj) => {
exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
// local db override
if (using_local_db) {
let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
let cursor = filter_obj ? exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
if (sort) {
cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
}
@@ -664,7 +664,7 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
exports.updateRecord = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
return true;
}
@@ -677,7 +677,7 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
exports.updateRecords = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
return true;
}
@@ -722,7 +722,7 @@ exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
return true;
}
@@ -733,7 +733,7 @@ exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
return true;
}
@@ -746,7 +746,7 @@ exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
exports.removeRecord = async (table, filter_obj) => {
// local db override
if (using_local_db) {
applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
return true;
}
@@ -757,7 +757,7 @@ exports.removeRecord = async (table, filter_obj) => {
// exports.removeRecordsByUIDBulk = async (table, uids) => {
// // local db override
// if (using_local_db) {
// applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
// exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
// return true;
// }
@@ -821,7 +821,7 @@ exports.removeAllRecords = async (table = null, filter_obj = null) => {
if (using_local_db) {
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
if (filter_obj) exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
else local_db.assign({[table_to_remove]: []}).write();
logger.debug(`Successfully removed records from ${table_to_remove}`);
}
@@ -1075,8 +1075,13 @@ exports.transferDB = async (local_to_remote) => {
This function is necessary to emulate mongodb's ability to search for null or missing values.
A filter of null or undefined for a property will find docs that have that property missing, or have it
null or undefined. We want that same functionality for the local DB as well
error: {$ne: null}
^ ^
| |
filter_prop filter_prop_value
*/
const applyFilterLocalDB = (db_path, filter_obj, operation) => {
exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
const filter_props = Object.keys(filter_obj);
const return_val = db_path[operation](record => {
if (!filter_props) return true;
@@ -1085,14 +1090,20 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
const filter_prop = filter_props[i];
const filter_prop_value = filter_obj[filter_prop];
if (filter_prop_value === undefined || filter_prop_value === null) {
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
filtered &= record[filter_prop] === undefined || record[filter_prop] === null;
} else {
if (typeof filter_prop_value === 'object') {
if (filter_prop_value['$regex']) {
if ('$regex' in filter_prop_value) {
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
} else if ('$ne' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
}
} else {
filtered &= record[filter_prop] === filter_prop_value;
// handle case of nested property check
if (filter_prop.includes('.'))
filtered &= utils.searchObjectByString(record, filter_prop) === filter_prop_value;
else
filtered &= record[filter_prop] === filter_prop_value;
}
}
}

View File

@@ -108,6 +108,7 @@ exports.clearDownload = async (download_uid) => {
}
async function handleDownloadError(download_uid, error_message) {
if (!download_uid) return;
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
}
@@ -186,7 +187,7 @@ async function collectInfo(download_uid) {
let args = await exports.generateArgs(url, type, options, download['user_uid']);
// get video info prior to download
let info = await getVideoInfoByURL(url, args, download_uid);
let info = await exports.getVideoInfoByURL(url, args, download_uid);
if (!info) {
// info failed, error presumably already recorded
@@ -203,9 +204,11 @@ async function collectInfo(download_uid) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
info = await getVideoInfoByURL(url, args, download_uid);
info = await exports.getVideoInfoByURL(url, args, download_uid);
}
download['category'] = category;
// setup info required to calculate download progress
const expected_file_size = utils.getExpectedFileSize(info);
@@ -507,7 +510,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
return downloadConfig;
}
async function getVideoInfoByURL(url, args = [], download_uid = null) {
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];

View File

@@ -148,6 +148,7 @@ exports.updateTaskSchedule = async (task_key, schedule) => {
await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
if (TASKS[task_key]['job']) {
TASKS[task_key]['job'].cancel();
TASKS[task_key]['job'] = null;
}
if (schedule) {
TASKS[task_key]['job'] = scheduleJob(task_key, schedule);

View File

@@ -42,6 +42,25 @@ const { uuid } = require('uuidv4');
db_api.initialize(db, users_db);
const sample_video_json = {
id: "Sample Video",
title: "Sample Video",
thumbnailURL: "https://sampleurl.jpg",
isAudio: false,
duration: 177.413,
url: "sampleurl.com",
uploader: "Sample Uploader",
size: 2838445,
path: "users\\admin\\video\\Sample Video.mp4",
upload_date: "2017-07-28",
description: null,
view_count: 230,
abr: 128,
thumbnailPath: null,
user_uid: "admin",
uid: "1ada04ab-2773-4dd4-bbdd-3e2d40761c50",
registered: 1628469039377
}
describe('Database', async function() {
describe('Import', async function() {
@@ -214,7 +233,7 @@ describe('Database', async function() {
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon: http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
@@ -235,6 +254,30 @@ describe('Database', async function() {
assert(success);
});
});
describe('Local DB Filters', async function() {
it('Basic', async function() {
const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: 'test'}, 'find');
assert(result && result['test'] === 'test');
});
it('Regex', async function() {
const filter = {$regex: `\\w+\\d`, $options: 'i'};
const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
assert(result && result['test'] === 'test1');
});
it('Not equals', async function() {
const filter = {$ne: 'test'};
const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
assert(result && result['test'] === 'test1');
});
it('Nested', async function() {
const result = db_api.applyFilterLocalDB([{test1: {test2: 'test3'}}, {test4: 'test5'}], {'test1.test2': 'test3'}, 'find');
assert(result && result['test1']['test2'] === 'test3');
});
})
});
describe('Multi User', async function() {
@@ -253,10 +296,12 @@ describe('Multi User', async function() {
assert(user);
});
});
describe('Video player - normal', function() {
const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
describe('Video player - normal', async function() {
await db_api.removeRecord('files', {uid: sample_video_json['uid']});
await db_api.insertRecordIntoTable('files', sample_video_json);
const video_to_test = sample_video_json['uid'];
it('Get video', async function() {
const video_obj = db_api.getVideo(video_to_test, 'admin');
const video_obj = await db_api.getVideo(video_to_test);
assert(video_obj);
});
@@ -341,7 +386,9 @@ describe('Downloader', function() {
});
it('Get file info', async function() {
this.timeout(300000);
const info = await downloader_api.getVideoInfoByURL(url);
assert(!!info);
});
it('Download file', async function() {
@@ -360,20 +407,23 @@ describe('Downloader', function() {
});
it('Pause file', async function() {
const returned_download = await downloader_api.createDownload(url, 'video', options);
await downloader_api.pauseDownload(returned_download['uid']);
const updated_download = await db_api.getRecord('download_queue', {uid: returned_download['uid']});
assert(updated_download['paused'] && !updated_download['running']);
});
it('Generate args', async function() {
const args = await downloader_api.generateArgs(url, 'video', options);
console.log(args);
assert(args.length > 0);
});
it('Generate args - subscription', async function() {
subscriptions_api.initialize(db_api, logger);
const sub = await subscriptions_api.getSubscription(sub_id);
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
console.log(args);
const args_normal = await downloader_api.generateArgs(url, 'video', options);
const args_sub = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
console.log(JSON.stringify(args_normal) !== JSON.stringify(args_sub));
});
it('Generate kodi NFO file', async function() {
@@ -417,7 +467,7 @@ describe('Tasks', function() {
};
tasks_api.TASKS['dummy_task'] = dummy_task;
await tasks_api.initialize();
await tasks_api.setupTasks();
});
it('Backup db', async function() {
const backups_original = await utils.recFindByExt('appdata', 'bak');
@@ -429,12 +479,13 @@ describe('Tasks', function() {
});
it('Check for missing files', async function() {
this.timeout(300000);
await db_api.removeAllRecords('files', {uid: 'test'});
const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
await db_api.insertRecordIntoTable('files', test_missing_file);
await tasks_api.executeTask('missing_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
assert(!missing_file_db_record, true);
});
it('Check for duplicate files', async function() {
@@ -447,10 +498,13 @@ describe('Tasks', function() {
await db_api.insertRecordIntoTable('files', test_duplicate_file1);
await db_api.insertRecordIntoTable('files', test_duplicate_file2);
await db_api.insertRecordIntoTable('files', test_duplicate_file3);
await tasks_api.executeTask('duplicate_files_check');
await tasks_api.executeRun('duplicate_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
await tasks_api.executeTask('duplicate_files_check');
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
assert(duplicated_record_count == 1, true);
});
@@ -475,22 +529,36 @@ describe('Tasks', function() {
});
it('Schedule and cancel task', async function() {
const today_4_hours = new Date();
today_4_hours.setHours(today_4_hours.getHours() + 4);
await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
this.timeout(5000);
const today_one_year = new Date();
today_one_year.setFullYear(today_one_year.getFullYear() + 1);
const schedule_obj = {
type: 'timestamp',
data: { timestamp: today_one_year.getTime() }
}
await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
const dummy_task = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(!!tasks_api.TASKS['dummy_task']['job']);
assert(!!dummy_task['schedule']);
await tasks_api.updateTaskSchedule('dummy_task', null);
assert(!!tasks_api.TASKS['dummy_task']['job'], false);
const dummy_task_updated = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(!tasks_api.TASKS['dummy_task']['job']);
assert(!dummy_task_updated['schedule']);
});
it('Schedule and run task', async function() {
this.timeout(5000);
const today_1_second = new Date();
today_1_second.setSeconds(today_1_second.getSeconds() + 1);
await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
const schedule_obj = {
type: 'timestamp',
data: { timestamp: today_1_second.getTime() }
}
await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
assert(!!tasks_api.TASKS['dummy_task']['job']);
await utils.wait(2000);
const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(dummy_task_obj['data'], true);
assert(dummy_task_obj['data']);
});
});

View File

@@ -456,6 +456,21 @@ function injectArgs(original_args, new_args) {
return updated_args;
}
const searchObjectByString = function(o, s) {
s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
s = s.replace(/^\./, ''); // strip a leading dot
var a = s.split('.');
for (var i = 0, n = a.length; i < n; ++i) {
var k = a[i];
if (k in o) {
o = o[k];
} else {
return;
}
}
return o;
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -489,7 +504,6 @@ module.exports = {
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
@@ -501,5 +515,6 @@ module.exports = {
fetchFile: fetchFile,
restartServer: restartServer,
injectArgs: injectArgs,
searchObjectByString: searchObjectByString,
File: File
}

View File

@@ -90,7 +90,7 @@ exports.updateYoutubeDL = async (latest_update_version) => {
exports.verifyBinaryExistsLinux = () => {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (!is_windows && details_json && details_json['path'] && details_json['path'].includes('.exe')) {
if (!is_windows && details_json && (details_json['path'].includes('.exe') || !details_json['path'])) {
details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
details_json['exec'] = 'youtube-dl';
details_json['version'] = OUTDATED_VERSION;