TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-31 17:10:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed issue where old DB backup didn't work

Browse Source 
Massive insertions to local DB are now split up into 30k chunks
This commit is contained in:
Isaac Abadi
2021-07-20 20:55:47 -06:00
parent 148ed9aa65
commit d10eb4f2eb
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/db.js
View File

@@ -616,7 +616,15 @@ exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
exports.insertRecordsIntoTable = async (table, docs, ignore_errors = false) => { exports.insertRecordsIntoTable = async (table, docs, ignore_errors = false) => {
// local db override // local db override
if (using_local_db) { if (using_local_db) {
local_db.get(table).push(...docs).write(); const records_limit = 30000;
if (docs.length < records_limit) {
local_db.get(table).push(...docs).write();
} else {
for (let i = 0; i < docs.length; i+=records_limit) {
const records_to_push = docs.slice(i, i+records_limit > docs.length ? docs.length : i+records_limit)
local_db.get(table).push(...records_to_push).write();
}
}
return true; return true;
} }
const output = await database.collection(table).insertMany(docs, {ordered: !ignore_errors}); const output = await database.collection(table).insertMany(docs, {ordered: !ignore_errors});
@@ -857,8 +865,8 @@ exports.generateJSONTables = async (db_json, users_json) => {
} }
exports.importJSONToDB = async (db_json, users_json) => { exports.importJSONToDB = async (db_json, users_json) => {
await fs.writeFile(`appdata/db.json.${Date.now()/1000}.bak`, JSON.stringify(db_json)); await fs.writeFile(`appdata/db.json.${Date.now()/1000}.bak`, JSON.stringify(db_json, null, 2));
await fs.writeFile(`appdata/users_db.json.${Date.now()/1000}.bak`, JSON.stringify(users_json)); await fs.writeFile(`appdata/users_db.json.${Date.now()/1000}.bak`, JSON.stringify(users_json, null, 2));
await exports.removeAllRecords(); await exports.removeAllRecords();
const tables_obj = await exports.generateJSONTables(db_json, users_json); const tables_obj = await exports.generateJSONTables(db_json, users_json);
@@ -869,7 +877,6 @@ exports.importJSONToDB = async (db_json, users_json) => {
for (let i = 0; i < table_keys.length; i++) { for (let i = 0; i < table_keys.length; i++) {
const table_key = table_keys[i]; const table_key = table_keys[i];
if (!tables_obj[table_key] || tables_obj[table_key].length === 0) continue; if (!tables_obj[table_key] || tables_obj[table_key].length === 0) continue;
console.log('hi');
success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key], true); success &= await exports.insertRecordsIntoTable(table_key, tables_obj[table_key], true);
} }