Added ability to backup remote DB

Added ability to restore DB
This commit is contained in:
Isaac Abadi
2022-04-21 19:29:50 -04:00
parent 091f81bb38
commit a288163644
19 changed files with 420 additions and 29 deletions

View File

@@ -947,6 +947,54 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/UpdateTaskScheduleRequest'
/api/updateTaskData:
post:
summary: Updates task data
operationId: post-api-update-task-data
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTaskDataRequest'
/api/getDBBackups:
post:
summary: Get database backups
operationId: post-api-get-database-backups
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/GetDBBackupsResponse'
requestBody:
content:
application/json:
schema:
type: object
/api/restoreDBBackup:
post:
summary: Restore database backup
operationId: post-api-restore-database-backup
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RestoreDBBackupRequest'
/api/auth/login:
post:
summary: Login
@@ -1524,6 +1572,16 @@ components:
required:
- task_key
- new_schedule
UpdateTaskDataRequest:
type: object
properties:
task_key:
type: string
new_data:
type: object
required:
- task_key
- new_data
GetTaskResponse:
type: object
properties:
@@ -1536,6 +1594,20 @@ components:
type: array
items:
$ref: '#/components/schemas/Task'
GetDBBackupsResponse:
type: object
properties:
tasks:
type: array
items:
$ref: '#/components/schemas/DBBackup'
RestoreDBBackupRequest:
type: object
required:
- file_name
properties:
file_name:
type: string
GetMp3sResponse:
required:
- mp3s
@@ -2328,6 +2400,25 @@ components:
type: number
timestamp:
type: number
DBBackup:
required:
- name
- timestamp
- size
- source
type: object
properties:
name:
type: string
timestamp:
type: number
size:
type: number
source:
type: string
enum:
- local
- remote
SubscriptionRequestData:
required:
- id

View File

@@ -1924,6 +1924,45 @@ app.post('/api/updateTaskSchedule', optionalJwt, async (req, res) => {
res.send({success: true});
});
app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const new_data = req.body.new_data;
const success = await db_api.updateRecord('tasks', {key: task_key}, {data: new_data});
res.send({success: success});
});
app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
const backup_dir = path.join('appdata', 'db_backup');
const db_backups = [];
const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false);
for (let i = 0; i < candidate_backups.length; i++) {
const candidate_backup = candidate_backups[i];
// must have specific format
if (candidate_backup.split('.').length - 1 !== 4) continue;
const candidate_backup_path = candidate_backup;
const stats = fs.statSync(candidate_backup_path);
db_backups.push({ name: path.basename(candidate_backup), timestamp: parseInt(candidate_backup.split('.')[2]), size: stats.size, source: candidate_backup.includes('local') ? 'local' : 'remote' });
}
db_backups.sort((a,b) => b.timestamp - a.timestamp);
res.send({db_backups: db_backups});
});
app.post('/api/restoreDBBackup', optionalJwt, async (req, res) => {
const file_name = req.body.file_name;
const success = await db_api.restoreDB(file_name);
res.send({success: success});
});
// logs management
app.post('/api/logs', optionalJwt, async function(req, res) {

View File

@@ -987,6 +987,52 @@ const createDownloadsRecords = (downloads) => {
return new_downloads;
}
exports.backupDB = async () => {
const backup_dir = path.join('appdata', 'db_backup');
fs.ensureDirSync(backup_dir);
const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
const path_to_backups = path.join(backup_dir, backup_file_name);
logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
table_to_records[table] = await exports.getRecords(table);
}
fs.writeJsonSync(path_to_backups, table_to_records);
return backup_file_name;
}
exports.restoreDB = async (file_name) => {
const path_to_backup = path.join('appdata', 'db_backup', file_name);
logger.debug('Reading database backup file.');
const table_to_records = fs.readJSONSync(path_to_backup);
if (!table_to_records) {
logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`);
return false;
}
logger.debug('Clearing database.');
await exports.removeAllRecords();
logger.debug('Database cleared! Beginning restore.');
let success = true;
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
}
logger.debug('Restore finished!');
return success;
}
exports.transferDB = async (local_to_remote) => {
const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) {
@@ -996,9 +1042,8 @@ exports.transferDB = async (local_to_remote) => {
using_local_db = !local_to_remote;
if (local_to_remote) {
// backup local DB
logger.debug('Backup up Local DB...');
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
logger.debug('Backup up DB...');
await exports.backupDB();
const db_connected = await exports.connectToDB(5, true);
if (!db_connected) {
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');

View File

@@ -7,8 +7,8 @@ const scheduler = require('node-schedule');
const TASKS = {
backup_local_db: {
run: utils.backupLocalDB,
title: 'Backup Local DB',
run: db_api.backupDB,
title: 'Backup DB',
job: null
},
missing_files_check: {
@@ -81,7 +81,8 @@ const setupTasks = async () => {
confirming: false,
data: null,
error: null,
schedule: null
schedule: null,
options: {}
});
} else {
// reset task if necessary

View File

@@ -70,6 +70,17 @@ describe('Database', async function() {
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Restore db', async function() {
const db_stats = await db_api.getDBStats();
const file_name = await db_api.backupDB();
await db_api.restoreDB(file_name);
const new_db_stats = await db_api.getDBStats();
assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats));
});
});
describe('Export', function() {
@@ -393,7 +404,7 @@ describe('Tasks', function() {
await tasks_api.initialize();
});
it('Backup local db', async function() {
it('Backup db', async function() {
const backups_original = await utils.recFindByExt('appdata', 'bak');
const original_length = backups_original.length;
await tasks_api.executeTask('backup_local_db');

View File

@@ -266,13 +266,7 @@ function getCurrentDownloader() {
return details_json['downloader'];
}
async function backupLocalDB() {
const path_to_backups = path.join('appdata', 'db_backup');
fs.ensureDir(path_to_backups);
await fs.copyFile('appdata/local_db.json', path.join(path_to_backups, `local_db.json.${Date.now()/1000}.bak`));
}
async function recFindByExt(base,ext,files,result)
async function recFindByExt(base, ext, files, result, recursive = true)
{
files = files || (await fs.readdir(base))
result = result || []
@@ -281,6 +275,7 @@ async function recFindByExt(base,ext,files,result)
var newbase = path.join(base,file)
if ( (await fs.stat(newbase)).isDirectory() )
{
if (!recursive) continue;
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
}
else
@@ -396,7 +391,6 @@ module.exports = {
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
getCurrentDownloader: getCurrentDownloader,
backupLocalDB: backupLocalDB,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,

View File

@@ -21,6 +21,7 @@ export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
export type { CropFileSettings } from './models/CropFileSettings';
export type { DatabaseFile } from './models/DatabaseFile';
export { DBBackup } from './models/DBBackup';
export type { DBInfoResponse } from './models/DBInfoResponse';
export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
@@ -45,6 +46,7 @@ export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
export type { GetDownloadRequest } from './models/GetDownloadRequest';
export type { GetDownloadResponse } from './models/GetDownloadResponse';
export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
@@ -74,6 +76,7 @@ export type { LoginResponse } from './models/LoginResponse';
export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse';
export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
export { Schedule } from './models/Schedule';
export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SharingToggle } from './models/SharingToggle';
@@ -98,6 +101,7 @@ export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentSt
export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
export type { UpdaterStatus } from './models/UpdaterStatus';
export type { UpdateServerRequest } from './models/UpdateServerRequest';
export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
export type { UpdateUserRequest } from './models/UpdateUserRequest';
export type { User } from './models/User';

View File

@@ -0,0 +1,21 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DBBackup {
name: string;
timestamp: number;
size: number;
source: DBBackup.source;
}
export namespace DBBackup {
export enum source {
LOCAL = 'local',
REMOTE = 'remote',
}
}

View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DBBackup } from './DBBackup';
export interface GetDBBackupsResponse {
tasks?: Array<DBBackup>;
}

View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface RestoreDBBackupRequest {
file_name: string;
}

View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface UpdateTaskDataRequest {
task_key: string;
new_data: any;
}

View File

@@ -90,6 +90,7 @@ import { ConcurrentStreamComponent } from './components/concurrent-stream/concur
import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component';
import { TasksComponent } from './components/tasks/tasks.component';
import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component';
registerLocaleData(es, 'es');
@@ -140,7 +141,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
ConcurrentStreamComponent,
SkipAdButtonComponent,
TasksComponent,
UpdateTaskScheduleDialogComponent
UpdateTaskScheduleDialogComponent,
RestoreDbDialogComponent
],
imports: [
CommonModule,

View File

@@ -48,15 +48,23 @@
<ng-container matColumnDef="actions">
<mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<div>
<ng-container *ngIf="element.data?.uids?.length > 0">
<button (click)="confirmTask(element.key)" [disabled]="element.running || element.confirming" mat-stroked-button>
<ng-container *ngIf="element.key == 'missing_files_check'" i18n="Clear missing files from DB">Clear missing files from DB:</ng-container>
<ng-container *ngIf="element.key == 'duplicate_files_check'" i18n="Clear duplicate files from DB">Clear duplicate files from DB:</ng-container>&nbsp;{{element.data.uids.length}}
</button>
</ng-container>
<button (click)="runTask(element.key)" [disabled]="element.running || element.confirming" mat-icon-button matTooltip="Run" i18n-matTooltip="Run"><mat-icon>play_arrow</mat-icon></button>
<button (click)="scheduleTask(element)" mat-icon-button matTooltip="Schedule" i18n-matTooltip="Schedule"><mat-icon>schedule</mat-icon></button>
<div class="container">
<div class="row justify-content-center">
<div *ngIf="element.data?.uids?.length > 0" class="col-12 mt-2">
<ng-container>
<button (click)="confirmTask(element.key)" [disabled]="element.running || element.confirming" mat-stroked-button>
<ng-container *ngIf="element.key == 'missing_files_check'" i18n="Clear missing files from DB">Clear missing files from DB:</ng-container>
<ng-container *ngIf="element.key == 'duplicate_files_check'" i18n="Clear duplicate files from DB">Clear duplicate files from DB:</ng-container>&nbsp;{{element.data.uids.length}}
</button>
</ng-container>
</div>
<div class="col-3" style="padding-right: 0px">
<button (click)="runTask(element.key)" [disabled]="element.running || element.confirming" mat-icon-button matTooltip="Run" i18n-matTooltip="Run"><mat-icon>play_arrow</mat-icon></button>
</div>
<div class="col-3" style="padding-left: 0px">
<button (click)="scheduleTask(element)" mat-icon-button matTooltip="Schedule" i18n-matTooltip="Schedule"><mat-icon>schedule</mat-icon></button>
</div>
</div>
</div>
</mat-cell>
</ng-container>
@@ -70,6 +78,8 @@
aria-label="Select page of tasks">
</mat-paginator>
</div>
<button style="margin-top: 10px; margin-left: 5px;" mat-stroked-button (click)="openRestoreDBBackupDialog()" i18n="Restore DB from backup button">Restore DB from backup</button>
</div>
<div *ngIf="(!tasks || tasks.length === 0) && tasks_retrieved">

View File

@@ -3,6 +3,7 @@ import { MatDialog } from '@angular/material/dialog';
import { MatPaginator } from '@angular/material/paginator';
import { MatSort } from '@angular/material/sort';
import { MatTableDataSource } from '@angular/material/table';
import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component';
import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
import { PostsService } from 'app/posts.services';
@@ -21,6 +22,8 @@ export class TasksComponent implements OnInit {
displayedColumns: string[] = ['title', 'last_ran', 'last_confirmed', 'status', 'actions'];
dataSource = null;
db_backups = [];
@ViewChild(MatPaginator) paginator: MatPaginator;
@ViewChild(MatSort) sort: MatSort;
@@ -70,12 +73,23 @@ export class TasksComponent implements OnInit {
runTask(task_key: string): void {
this.postsService.runTask(task_key).subscribe(res => {
this.getTasks();
this.getDBBackups();
if (res['success']) this.postsService.openSnackBar($localize`Successfully ran task!`);
else this.postsService.openSnackBar($localize`Failed to run task!`);
}, err => {
this.postsService.openSnackBar($localize`Failed to run task!`);
console.error(err);
});
}
confirmTask(task_key: string): void {
this.postsService.confirmTask(task_key).subscribe(res => {
this.getTasks();
if (res['success']) this.postsService.openSnackBar($localize`Successfully confirmed task!`);
else this.postsService.openSnackBar($localize`Failed to confirm task!`);
}, err => {
this.postsService.openSnackBar($localize`Failed to confirm task!`);
console.error(err);
});
}
@@ -96,6 +110,21 @@ export class TasksComponent implements OnInit {
});
}
getDBBackups(): void {
this.postsService.getDBBackups().subscribe(res => {
this.db_backups = res['db_backups'];
});
}
openRestoreDBBackupDialog(): void {
this.dialog.open(RestoreDbDialogComponent, {
data: {
db_backups: this.db_backups
},
width: '80vw'
})
}
}
export interface Task {

View File

@@ -0,0 +1,29 @@
<h4 mat-dialog-title><ng-container i18n="Restore DB from backup">Restore DB from backup</ng-container></h4>
<mat-dialog-content>
<mat-selection-list [multiple]="false" [(ngModel)]="selected_backup">
<mat-list-option *ngFor="let db_backup of db_backups" [value]="db_backup.name" [matTooltip]="db_backup.name">
<div class="container-fluid">
<div class="row">
<div class="col-4">
{{db_backup.timestamp*1000 | date: 'short'}}
</div>
<div class="col-4">
{{(db_backup.size/1000).toFixed(2)}} kB
</div>
<div class="col-4" style="text-transform: capitalize;">
{{db_backup.source}}
</div>
</div>
</div>
</mat-list-option>
</mat-selection-list>
</mat-dialog-content>
<mat-dialog-actions>
<button mat-button mat-dialog-close><ng-container i18n="Restore DB cancel button">Cancel</ng-container></button>
<button mat-button [disabled]="restoring" (click)="restoreClicked()" [disabled]="!selected_backup || selected_backup.length !== 1"><ng-container i18n="Restore button">Restore</ng-container></button>
<div class="mat-spinner" *ngIf="restoring">
<mat-spinner [diameter]="25"></mat-spinner>
</div>
</mat-dialog-actions>

View File

@@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { RestoreDbDialogComponent } from './restore-db-dialog.component';
describe('RestoreDbDialogComponent', () => {
let component: RestoreDbDialogComponent;
let fixture: ComponentFixture<RestoreDbDialogComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ RestoreDbDialogComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(RestoreDbDialogComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

View File

@@ -0,0 +1,47 @@
import { Component, Inject, OnInit } from '@angular/core';
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-restore-db-dialog',
templateUrl: './restore-db-dialog.component.html',
styleUrls: ['./restore-db-dialog.component.scss']
})
export class RestoreDbDialogComponent implements OnInit {
db_backups = [];
selected_backup = null;
restoring = false;
constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<RestoreDbDialogComponent>, private postsService: PostsService) {
if (this.data?.db_backups) {
this.db_backups = this.data.db_backups;
}
this.getDBBackups();
}
ngOnInit(): void {
}
getDBBackups(): void {
this.postsService.getDBBackups().subscribe(res => {
this.db_backups = res['db_backups'];
});
}
restoreClicked(): void {
if (this.selected_backup.length !== 1) return;
this.postsService.restoreDBBackup(this.selected_backup[0]).subscribe(res => {
if (res['success']) {
this.postsService.openSnackBar('Database successfully restored!');
} else {
this.postsService.openSnackBar('Failed to restore database! See logs for more info.');
}
}, err => {
this.postsService.openSnackBar('Failed to restore database! See browser console for more info.');
console.error(err);
});
}
}

View File

@@ -93,6 +93,9 @@ import {
GetTaskRequest,
GetTaskResponse,
UpdateTaskScheduleRequest,
UpdateTaskDataRequest,
RestoreDBBackupRequest,
Schedule,
} from '../api-types';
import { isoLangs } from './settings/locales_list';
import { Title } from '@angular/platform-browser';
@@ -588,26 +591,40 @@ export class PostsService implements CanActivate {
return this.http.post<SuccessObject>(this.path + 'getTasks', {}, this.httpOptions);
}
getTask(task_key) {
getTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<GetTaskResponse>(this.path + 'getTask', body, this.httpOptions);
}
runTask(task_key) {
runTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<SuccessObject>(this.path + 'runTask', body, this.httpOptions);
}
confirmTask(task_key) {
confirmTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<SuccessObject>(this.path + 'confirmTask', body, this.httpOptions);
}
updateTaskSchedule(task_key, schedule) {
updateTaskSchedule(task_key: string, schedule: Schedule) {
const body: UpdateTaskScheduleRequest = {task_key: task_key, new_schedule: schedule};
return this.http.post<SuccessObject>(this.path + 'updateTaskSchedule', body, this.httpOptions);
}
updateTaskData(task_key: string, data: any) {
const body: UpdateTaskDataRequest = {task_key: task_key, new_data: data};
return this.http.post<SuccessObject>(this.path + 'updateTaskData', body, this.httpOptions);
}
getDBBackups() {
return this.http.post<SuccessObject>(this.path + 'getDBBackups', {}, this.httpOptions);
}
restoreDBBackup(file_name: string) {
const body: RestoreDBBackupRequest = {file_name: file_name};
return this.http.post<SuccessObject>(this.path + 'restoreDBBackup', body, this.httpOptions);
}
getVersionInfo() {
return this.http.get<VersionInfoResponse>(this.path + 'versionInfo', this.httpOptions);
}