TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 20:10:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improved DB tests, now both local and remote DB can be tested easily

Browse Source
This commit is contained in:
Tzahi12345
2023-01-16 02:42:36 -05:00
parent f20a31ed0f
commit c724a8019a
4 changed files with 171 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.vscode/settings.json vendored
View File

@@ -1,5 +1,8 @@
{ {
"mochaExplorer.files": "backend/test/**/*.js", "mochaExplorer.files": "backend/test/**/*.js",
"mochaExplorer.cwd": "backend", "mochaExplorer.cwd": "backend",
"mochaExplorer.globImplementation": "vscode" "mochaExplorer.globImplementation": "vscode",
"mochaExplorer.env": {
"YTDL_MODE": "debug"
}
} }

8
backend/db.js
View File

@@ -2,6 +2,7 @@ var fs = require('fs-extra')
var path = require('path') var path = require('path')
const { MongoClient } = require("mongodb"); const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config'); const config_api = require('./config');
var utils = require('./utils') var utils = require('./utils')
@@ -152,6 +153,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
await database.collection(table).createIndex(text_search); await database.collection(table).createIndex(text_search);
} }
}); });
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
return true; return true;
} catch(err) { } catch(err) {
logger.error(err); logger.error(err);
@@ -578,7 +580,6 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => { exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override // local db override
if (using_local_db) { if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write(); local_db.get(table).push(doc).write();
return true; return true;
} }
@@ -1141,3 +1142,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
}); });
return return_val; return return_val;
} }
// should only be used for tests
exports.setLocalDBMode = (mode) => {
using_local_db = mode;
}

1
backend/package.json
View File

@@ -32,6 +32,7 @@
"fs-extra": "^9.0.0", "fs-extra": "^9.0.0",
"gotify": "^1.1.0", "gotify": "^1.1.0",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0", "lowdb": "^1.0.0",
"md5": "^2.2.1", "md5": "^2.2.1",
"merge-files": "^0.1.2", "merge-files": "^0.1.2",

290
backend/test/tests.js
View File

@@ -104,151 +104,179 @@ describe('Database', async function() {
}); });
describe('Basic functions', async function() { describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB(); // test both local_db and remote_db
await db_api.removeAllRecords('test'); const local_db_modes = [false, true];
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Find duplicates by key', async function() { for (const local_db_mode of local_db_modes) {
const test_duplicates = [ let use_local_db = local_db_mode;
{ describe(`Use local DB - ${use_local_db}`, async function() {
test: 'testing', beforeEach(async function() {
key: '1' if (!use_local_db) {
}, this.timeout(120000);
{ await db_api.connectToDB(0);
test: 'testing', }
key: '2' await db_api.removeAllRecords('test');
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
}); });
} it('Add and read record', async function() {
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records); this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const received_records = await db_api.getRecords('test'); const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); assert(added_record['test_add'] === 'test');
}); await db_api.removeRecord('test', {test_add: 'test'});
});
it('Bulk update', async function() { it('Add and read record - Nested property', async function() {
// bulk add records this.timeout(120000);
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const test_records = []; const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
const update_obj = {}; const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { assert(added_record['test_add'] === 'test');
const test_uid = uuid(); assert(!not_added_record);
test_records.push({ await db_api.removeRecord('test', {test_add: 'test'});
uid: test_uid });
it('Replace filter', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
assert(count === 1);
await db_api.removeRecord('test', {test_replace_filter: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
}); });
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added it('Update record', async function() {
const received_records = await db_api.getRecords('test'); await db_api.insertRecordIntoTable('test', {test_update: 'test'});
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj); it('Remove record', async function() {
assert(success); await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
const received_updated_records = await db_api.getRecords('test'); it('Push to record array', async function() {
for (let i = 0; i < received_updated_records.length; i++) { await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
success &= received_updated_records[i]['added_field']; await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
} const record = await db_api.getRecord('test', {test: 'test'});
assert(success); assert(record);
}); assert(record['test_array'].length === 1);
});
it('Stats', async function() { it('Pull from record array', async function() {
const stats = await db_api.getDBStats(); await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
assert(stats); await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
}); const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Query speed', async function() { it('Bulk add', async function() {
this.timeout(120000); this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000 const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = []; const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943'; for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { test_records.push({
const uid = uuid(); uid: uuid()
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid; });
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632}); }
} const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`); const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
const query_start = Date.now(); it('Bulk update', async function() {
const random_record = await db_api.getRecord('test', {uid: random_uid}); // bulk add records
const query_end = Date.now(); const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
console.log(random_record) // makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
console.log(`Query time: ${(query_end - query_start)/1000}s`); success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
success = !!random_record; const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
assert(success); it('Stats', async function() {
}); const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
}
}); });
describe('Local DB Filters', async function() { describe('Local DB Filters', async function() {