mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-07 12:00:01 +03:00
Improved DB tests, now both local and remote DB can be tested easily
This commit is contained in:
Tzahi12345
5
.vscode/settings.json
vendored
5
.vscode/settings.json
vendored
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"mochaExplorer.files": "backend/test/**/*.js",
|
||||
"mochaExplorer.cwd": "backend",
|
||||
"mochaExplorer.globImplementation": "vscode"
|
||||
"mochaExplorer.globImplementation": "vscode",
|
||||
"mochaExplorer.env": {
|
||||
"YTDL_MODE": "debug"
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ var fs = require('fs-extra')
|
||||
var path = require('path')
|
||||
const { MongoClient } = require("mongodb");
|
||||
const { uuid } = require('uuidv4');
|
||||
const _ = require('lodash');
|
||||
|
||||
const config_api = require('./config');
|
||||
var utils = require('./utils')
|
||||
@@ -152,6 +153,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
|
||||
await database.collection(table).createIndex(text_search);
|
||||
}
|
||||
});
|
||||
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
|
||||
return true;
|
||||
} catch(err) {
|
||||
logger.error(err);
|
||||
@@ -578,7 +580,6 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
|
||||
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
|
||||
// local db override
|
||||
if (using_local_db) {
|
||||
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
|
||||
local_db.get(table).push(doc).write();
|
||||
return true;
|
||||
}
|
||||
@@ -1141,3 +1142,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
|
||||
});
|
||||
return return_val;
|
||||
}
|
||||
|
||||
// should only be used for tests
|
||||
exports.setLocalDBMode = (mode) => {
|
||||
using_local_db = mode;
|
||||
}
|
||||
@@ -32,6 +32,7 @@
|
||||
"fs-extra": "^9.0.0",
|
||||
"gotify": "^1.1.0",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"lodash": "^4.17.21",
|
||||
"lowdb": "^1.0.0",
|
||||
"md5": "^2.2.1",
|
||||
"merge-files": "^0.1.2",
|
||||
|
||||
@@ -104,151 +104,179 @@ describe('Database', async function() {
|
||||
});
|
||||
|
||||
describe('Basic functions', async function() {
|
||||
beforeEach(async function() {
|
||||
await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('test');
|
||||
});
|
||||
it('Add and read record', async function() {
|
||||
this.timeout(120000);
|
||||
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
|
||||
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
|
||||
assert(added_record['test_add'] === 'test');
|
||||
await db_api.removeRecord('test', {test_add: 'test'});
|
||||
});
|
||||
|
||||
// test both local_db and remote_db
|
||||
const local_db_modes = [false, true];
|
||||
|
||||
it('Find duplicates by key', async function() {
|
||||
const test_duplicates = [
|
||||
{
|
||||
test: 'testing',
|
||||
key: '1'
|
||||
},
|
||||
{
|
||||
test: 'testing',
|
||||
key: '2'
|
||||
},
|
||||
{
|
||||
test: 'testing_missing',
|
||||
key: '3'
|
||||
},
|
||||
{
|
||||
test: 'testing',
|
||||
key: '4'
|
||||
}
|
||||
];
|
||||
await db_api.insertRecordsIntoTable('test', test_duplicates);
|
||||
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
|
||||
console.log(duplicates);
|
||||
});
|
||||
|
||||
it('Update record', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
|
||||
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
|
||||
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
|
||||
assert(updated_record['added_field']);
|
||||
await db_api.removeRecord('test', {test_update: 'test'});
|
||||
});
|
||||
|
||||
it('Remove record', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
|
||||
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
|
||||
assert(delete_succeeded);
|
||||
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
|
||||
assert(!deleted_record);
|
||||
});
|
||||
|
||||
it('Push to record array', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
|
||||
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
|
||||
const record = await db_api.getRecord('test', {test: 'test'});
|
||||
assert(record);
|
||||
assert(record['test_array'].length === 1);
|
||||
});
|
||||
|
||||
it('Pull from record array', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
|
||||
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
|
||||
const record = await db_api.getRecord('test', {test: 'test'});
|
||||
assert(record);
|
||||
assert(record['test_array'].length === 0);
|
||||
});
|
||||
|
||||
it('Bulk add', async function() {
|
||||
this.timeout(120000);
|
||||
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
test_records.push({
|
||||
uid: uuid()
|
||||
for (const local_db_mode of local_db_modes) {
|
||||
let use_local_db = local_db_mode;
|
||||
describe(`Use local DB - ${use_local_db}`, async function() {
|
||||
beforeEach(async function() {
|
||||
if (!use_local_db) {
|
||||
this.timeout(120000);
|
||||
await db_api.connectToDB(0);
|
||||
}
|
||||
await db_api.removeAllRecords('test');
|
||||
});
|
||||
}
|
||||
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
|
||||
const received_records = await db_api.getRecords('test');
|
||||
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
|
||||
});
|
||||
|
||||
it('Bulk update', async function() {
|
||||
// bulk add records
|
||||
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
const update_obj = {};
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
const test_uid = uuid();
|
||||
test_records.push({
|
||||
uid: test_uid
|
||||
it('Add and read record', async function() {
|
||||
this.timeout(120000);
|
||||
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
|
||||
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
|
||||
assert(added_record['test_add'] === 'test');
|
||||
await db_api.removeRecord('test', {test_add: 'test'});
|
||||
});
|
||||
it('Add and read record - Nested property', async function() {
|
||||
this.timeout(120000);
|
||||
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
|
||||
const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
|
||||
const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
|
||||
assert(added_record['test_add'] === 'test');
|
||||
assert(!not_added_record);
|
||||
await db_api.removeRecord('test', {test_add: 'test'});
|
||||
});
|
||||
it('Replace filter', async function() {
|
||||
this.timeout(120000);
|
||||
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
|
||||
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
|
||||
const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
|
||||
assert(count === 1);
|
||||
await db_api.removeRecord('test', {test_replace_filter: 'test'});
|
||||
});
|
||||
it('Find duplicates by key', async function() {
|
||||
const test_duplicates = [
|
||||
{
|
||||
test: 'testing',
|
||||
key: '1'
|
||||
},
|
||||
{
|
||||
test: 'testing',
|
||||
key: '2'
|
||||
},
|
||||
{
|
||||
test: 'testing_missing',
|
||||
key: '3'
|
||||
},
|
||||
{
|
||||
test: 'testing',
|
||||
key: '4'
|
||||
}
|
||||
];
|
||||
await db_api.insertRecordsIntoTable('test', test_duplicates);
|
||||
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
|
||||
console.log(duplicates);
|
||||
});
|
||||
update_obj[test_uid] = {added_field: true};
|
||||
}
|
||||
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
assert(success);
|
||||
|
||||
// makes sure they are added
|
||||
const received_records = await db_api.getRecords('test');
|
||||
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
|
||||
it('Update record', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
|
||||
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
|
||||
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
|
||||
assert(updated_record['added_field']);
|
||||
await db_api.removeRecord('test', {test_update: 'test'});
|
||||
});
|
||||
|
||||
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
|
||||
assert(success);
|
||||
it('Remove record', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
|
||||
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
|
||||
assert(delete_succeeded);
|
||||
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
|
||||
assert(!deleted_record);
|
||||
});
|
||||
|
||||
const received_updated_records = await db_api.getRecords('test');
|
||||
for (let i = 0; i < received_updated_records.length; i++) {
|
||||
success &= received_updated_records[i]['added_field'];
|
||||
}
|
||||
assert(success);
|
||||
});
|
||||
it('Push to record array', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
|
||||
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
|
||||
const record = await db_api.getRecord('test', {test: 'test'});
|
||||
assert(record);
|
||||
assert(record['test_array'].length === 1);
|
||||
});
|
||||
|
||||
it('Stats', async function() {
|
||||
const stats = await db_api.getDBStats();
|
||||
assert(stats);
|
||||
});
|
||||
it('Pull from record array', async function() {
|
||||
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
|
||||
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
|
||||
const record = await db_api.getRecord('test', {test: 'test'});
|
||||
assert(record);
|
||||
assert(record['test_array'].length === 0);
|
||||
});
|
||||
|
||||
it('Query speed', async function() {
|
||||
this.timeout(120000);
|
||||
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
const uid = uuid();
|
||||
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
|
||||
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
|
||||
}
|
||||
const insert_start = Date.now();
|
||||
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
const insert_end = Date.now();
|
||||
it('Bulk add', async function() {
|
||||
this.timeout(120000);
|
||||
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
test_records.push({
|
||||
uid: uuid()
|
||||
});
|
||||
}
|
||||
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
|
||||
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
|
||||
const received_records = await db_api.getRecords('test');
|
||||
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
|
||||
});
|
||||
|
||||
const query_start = Date.now();
|
||||
const random_record = await db_api.getRecord('test', {uid: random_uid});
|
||||
const query_end = Date.now();
|
||||
it('Bulk update', async function() {
|
||||
// bulk add records
|
||||
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
const update_obj = {};
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
const test_uid = uuid();
|
||||
test_records.push({
|
||||
uid: test_uid
|
||||
});
|
||||
update_obj[test_uid] = {added_field: true};
|
||||
}
|
||||
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
assert(success);
|
||||
|
||||
console.log(random_record)
|
||||
// makes sure they are added
|
||||
const received_records = await db_api.getRecords('test');
|
||||
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
|
||||
|
||||
console.log(`Query time: ${(query_end - query_start)/1000}s`);
|
||||
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
|
||||
assert(success);
|
||||
|
||||
success = !!random_record;
|
||||
const received_updated_records = await db_api.getRecords('test');
|
||||
for (let i = 0; i < received_updated_records.length; i++) {
|
||||
success &= received_updated_records[i]['added_field'];
|
||||
}
|
||||
assert(success);
|
||||
});
|
||||
|
||||
assert(success);
|
||||
});
|
||||
it('Stats', async function() {
|
||||
const stats = await db_api.getDBStats();
|
||||
assert(stats);
|
||||
});
|
||||
|
||||
it('Query speed', async function() {
|
||||
this.timeout(120000);
|
||||
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
|
||||
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
|
||||
const uid = uuid();
|
||||
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
|
||||
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
|
||||
}
|
||||
const insert_start = Date.now();
|
||||
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
|
||||
const insert_end = Date.now();
|
||||
|
||||
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
|
||||
|
||||
const query_start = Date.now();
|
||||
const random_record = await db_api.getRecord('test', {uid: random_uid});
|
||||
const query_end = Date.now();
|
||||
|
||||
console.log(random_record)
|
||||
|
||||
console.log(`Query time: ${(query_end - query_start)/1000}s`);
|
||||
|
||||
success = !!random_record;
|
||||
|
||||
assert(success);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('Local DB Filters', async function() {
|
||||
|
||||
Reference in New Issue
Block a user