Changed DB structure again

Added support for MongoDB Added tests relating to new DB system Category rules are now case insensitive Fixed playlist modification change state
2026-03-18 02:30:55 +03:00 · 2021-07-16 00:05:08 -06:00
parent 984e990103
commit bb6503e86d
27 changed files with 1426 additions and 699 deletions
--- a/src/app/components/downloads/downloads.component.html
+++ b/src/app/components/downloads/downloads.component.html
@@ -1,21 +1,21 @@
 <div style="padding: 20px;">
-    <div *ngFor="let session_downloads of downloads | keyvalue">
-        <ng-container *ngIf="keys(session_downloads.value).length > 0">
+    <div *ngFor="let session_downloads of downloads">
+        <ng-container *ngIf="keys(session_downloads).length > 2">
            <mat-card style="padding-bottom: 30px; margin-bottom: 15px;">
-                <h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads.key}}
-                    <span *ngIf="session_downloads.key === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
+                <h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads['session_id']}}
+                    <span *ngIf="session_downloads['session_id'] === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
                </h4>
                <div class="container">
                    <div class="row">
-                        <div *ngFor="let download of session_downloads.value | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
-                            <mat-card *ngIf="download.value" class="mat-elevation-z3">
-                                <app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads.key, download.value.uid)"></app-download-item>
+                        <div *ngFor="let download of session_downloads | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
+                            <mat-card *ngIf="download.key !== 'session_id' && download.key !== '_id' && download.value" class="mat-elevation-z3">
+                                <app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads['session_id'], download.value.uid)"></app-download-item>
                            </mat-card>
                        </div>
                    </div>
                </div>
                <div>
-                    <button style="top: 15px;" (click)="clearDownloads(session_downloads.key)" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
+                    <button style="top: 15px;" (click)="clearDownloads(session_downloads['session_id'])" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
                </div>
            </mat-card>
        </ng-container>
--- a/src/app/components/downloads/downloads.component.ts
+++ b/src/app/components/downloads/downloads.component.ts
@@ -35,7 +35,7 @@ import { Router } from '@angular/router';
 export class DownloadsComponent implements OnInit, OnDestroy {

  downloads_check_interval = 1000;
-  downloads = {};
+  downloads = [];
  interval_id = null;

  keys = Object.keys;
@@ -137,6 +137,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
        this.downloads[session_id] = session_downloads_by_id;
      } else {
        for (let j = 0; j < session_download_ids.length; j++) {
+          if (session_download_ids[j] === 'session_id' || session_download_ids[j] === '_id') continue;
          const download_id = session_download_ids[j];
          const download = new_downloads_by_session[session_id][download_id]
          if (!this.downloads[session_id][download_id]) {
@@ -156,11 +157,10 @@ export class DownloadsComponent implements OnInit, OnDestroy {

  downloadsValid() {
    let valid = false;
-    const keys = this.keys(this.downloads);
-    for (let i = 0; i < keys.length; i++) {
-      const key = keys[i];
-      const value = this.downloads[key];
-      if (this.keys(value).length > 0) {
+    for (let i = 0; i < this.downloads.length; i++) {
+      const session_downloads = this.downloads[i];
+      if (!session_downloads) continue;
+      if (this.keys(session_downloads).length > 2) {
        valid = true;
        break;
      }
--- a/src/app/components/manage-role/manage-role.component.html
+++ b/src/app/components/manage-role/manage-role.component.html
@@ -5,7 +5,7 @@
        <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
            <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
            <span matLine>
-                <mat-radio-group [disabled]="permission === 'settings' && role.name === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
+                <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
                </mat-radio-group>
--- a/src/app/components/manage-role/manage-role.component.ts
+++ b/src/app/components/manage-role/manage-role.component.ts
@@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit {
  }

  changeRolePermissions(change, permission) {
-    this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => {
+    this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => {
      if (res['success']) {

      } else {
--- a/src/app/components/modify-users/modify-users.component.html
+++ b/src/app/components/modify-users/modify-users.component.html
@@ -94,7 +94,7 @@
    </div>
    <button color="primary" [matMenuTriggerFor]="edit_roles_menu" class="edit-role" mat-raised-button><ng-container i18n="Edit role">Edit Role</ng-container></button>
    <mat-menu #edit_roles_menu="matMenu">
-        <button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.name}}</button>
+        <button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.key}}</button>
    </mat-menu>
    </div>
    
--- a/src/app/components/modify-users/modify-users.component.ts
+++ b/src/app/components/modify-users/modify-users.component.ts
@@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {

  getRoles() {
    this.postsService.getRoles().subscribe(res => {
-      this.roles = [];
-      const roles = res['roles'];
-      const role_names = Object.keys(roles);
-      for (let i = 0; i < role_names.length; i++) {
-        const role_name = role_names[i];
-        this.roles.push({
-          name: role_name,
-          permissions: roles[role_name]['permissions']
-        });
-      }
+      this.roles = res['roles'];
    });
  }

--- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts
+++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts
@@ -61,7 +61,7 @@ export class ModifyPlaylistComponent implements OnInit {
  }

  playlistChanged() {
-    return JSON.stringify(this.playlist) === JSON.stringify(this.original_playlist);
+    return JSON.stringify(this.playlist) !== JSON.stringify(this.original_playlist);
  }

  getPlaylist() {
--- a/src/app/main/main.component.html
+++ b/src/app/main/main.component.html
@@ -20,11 +20,16 @@
                                        </ng-container>
                                    </mat-label>
                                    <mat-select [ngModelOptions]="{standalone: true}" [(ngModel)]="selectedQuality">
-                                      <ng-container *ngFor="let option of qualityOptions[(audioOnly) ? 'audio' : 'video']">
-                                        <mat-option *ngIf="option.value === '' || url && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats'] && cachedAvailableFormats[url]['formats'][(audioOnly) ? 'audio' : 'video'][option.value]" [value]="option.value">
-                                            {{option.label}}
+                                        <mat-option [value]="''">
+                                            Max
                                        </mat-option>
-                                      </ng-container>
+                                        <ng-container *ngIf="url && cachedAvailableFormats && cachedAvailableFormats[url]?.formats">
+                                            <ng-container *ngFor="let option of cachedAvailableFormats[url]['formats'][audioOnly ? 'audio' : 'video']">
+                                                <mat-option *ngIf="option.key !== 'best_audio_format'" [value]="option">
+                                                    {{option.key}}
+                                                </mat-option>
+                                            </ng-container>
+                                        </ng-container>
                                    </mat-select>
                                    <div class="spinner-div" *ngIf="url !== '' && cachedAvailableFormats[url] && cachedAvailableFormats[url]['formats_loading']">
                                        <mat-spinner [diameter]="25"></mat-spinner>
--- a/src/app/main/main.component.ts
+++ b/src/app/main/main.component.ts
@@ -500,23 +500,26 @@ export class MainComponent implements OnInit {
  }

  getSelectedAudioFormat() {
-    if (this.selectedQuality === '') { return null };
+    if (this.selectedQuality === '') { return null; }
    const cachedFormatsExists = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
    if (cachedFormatsExists) {
      const audio_formats = this.cachedAvailableFormats[this.url]['formats']['audio'];
-      return audio_formats[this.selectedQuality]['format_id'];
+      return this.selectedQuality['format_id'];
    } else {
      return null;
    }
  }

  getSelectedVideoFormat() {
-    if (this.selectedQuality === '') { return null };
-    const cachedFormatsExists = this.cachedAvailableFormats[this.url] &&  this.cachedAvailableFormats[this.url]['formats'];
-    if (cachedFormatsExists) {
-      const video_formats = this.cachedAvailableFormats[this.url]['formats']['video'];
-      if (video_formats['best_audio_format'] && this.selectedQuality !== '') {
-          return video_formats[this.selectedQuality]['format_id'] + '+' + video_formats['best_audio_format'];
+    if (this.selectedQuality === '') { return null; }
+    const cachedFormats = this.cachedAvailableFormats[this.url] && this.cachedAvailableFormats[this.url]['formats'];
+    if (cachedFormats) {
+      const video_formats = cachedFormats['video'];
+      if (this.selectedQuality) {
+        let selected_video_format = this.selectedQuality['format_id'];
+        // add in audio format if necessary
+        if (!this.selectedQuality['acodec'] && cachedFormats['best_audio_format']) selected_video_format += `+${cachedFormats['best_audio_format']}`;
+        return selected_video_format;
      }
    }
    return null;
@@ -644,9 +647,8 @@ export class MainComponent implements OnInit {
          this.errorFormats(url);
          return;
        }
-        const parsed_infos = this.getAudioAndVideoFormats(infos.formats);
-        const available_formats = {audio: parsed_infos[0], video: parsed_infos[1]};
-        this.cachedAvailableFormats[url]['formats'] = available_formats;
+        this.cachedAvailableFormats[url]['formats'] = this.getAudioAndVideoFormats(infos.formats);
+        console.log(this.cachedAvailableFormats[url]['formats']);
      }, err => {
        this.errorFormats(url);
      });
@@ -689,7 +691,7 @@ export class MainComponent implements OnInit {
      if (audio_format) {
        format_array.push('-f', audio_format);
      } else if (this.selectedQuality) {
-        format_array.push('--audio-quality', this.selectedQuality);
+        format_array.push('--audio-quality', this.selectedQuality['format_id']);
      }

      // pushes formats
@@ -705,7 +707,7 @@ export class MainComponent implements OnInit {
      if (video_format) {
        format_array = ['-f', video_format];
      } else if (this.selectedQuality) {
-        format_array = [`bestvideo[height=${this.selectedQuality}]+bestaudio/best[height=${this.selectedQuality}]`];
+        format_array = [`bestvideo[height=${this.selectedQuality['format_id']}]+bestaudio/best[height=${this.selectedQuality}]`];
      }

      // pushes formats
@@ -802,9 +804,11 @@ export class MainComponent implements OnInit {
    }
  }

-  getAudioAndVideoFormats(formats): any[] {
-    const audio_formats = {};
-    const video_formats = {};
+  getAudioAndVideoFormats(formats) {
+    const audio_formats: any = {};
+    const video_formats: any = {};
+
+    console.log(formats);

    for (let i = 0; i < formats.length; i++) {
      const format_obj = {type: null};
@@ -815,9 +819,12 @@ export class MainComponent implements OnInit {
      format_obj.type = format_type;
      if (format_obj.type === 'audio' && format.abr) {
        const key = format.abr.toString() + 'K';
+        format_obj['key'] = key;
        format_obj['bitrate'] = format.abr;
        format_obj['format_id'] = format.format_id;
        format_obj['ext'] = format.ext;
+        format_obj['label'] = key;
+
        // don't overwrite if not m4a
        if (audio_formats[key]) {
          if (format.ext === 'm4a') {
@@ -828,11 +835,14 @@ export class MainComponent implements OnInit {
        }
      } else if (format_obj.type === 'video') {
        // check if video format is mp4
-        const key = format.format_note.replace('p', '');
+        const key = `${format.height}p${Math.round(format.fps)}`;
        if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') {
+          format_obj['key'] = key;
          format_obj['height'] = format.height;
          format_obj['acodec'] = format.acodec;
          format_obj['format_id'] = format.format_id;
+          format_obj['label'] = key;
+          format_obj['fps'] = Math.round(format.fps);

          // no acodec means no overwrite
          if (!(video_formats[key]) || format_obj['acodec'] !== 'none') {
@@ -842,9 +852,17 @@ export class MainComponent implements OnInit {
      }
    }

-    video_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats);
+    const parsed_formats: any = {};

-    return [audio_formats, video_formats]
+    parsed_formats['best_audio_format'] = this.getBestAudioFormatForMp4(audio_formats);
+
+    parsed_formats['video'] = Object.values(video_formats);
+    parsed_formats['audio'] = Object.values(audio_formats);
+
+    parsed_formats['video'] = parsed_formats['video'].sort((a, b) => b.height - a.height || b.fps - a.fps);
+    parsed_formats['audio'] = parsed_formats['audio'].sort((a, b) => b.bitrate - a.bitrate);
+
+    return parsed_formats;
  }

  getBestAudioFormatForMp4(audio_formats) {
--- a/src/app/player/player.component.html
+++ b/src/app/player/player.component.html
@@ -27,11 +27,11 @@
                                    </ng-container>
                                </div>
                                <div class="col-2">
-                                    <ng-container *ngIf="playlist.length > 1">
+                                    <ng-container *ngIf="db_playlist">
                                        <button (click)="downloadContent()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
                                        <button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
                                    </ng-container>
-                                    <ng-container *ngIf="playlist.length === 1">
+                                    <ng-container *ngIf="db_file">
                                        <button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
                                        <button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
                                        <button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button>
--- a/src/app/player/player.component.ts
+++ b/src/app/player/player.component.ts
@@ -171,7 +171,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
            console.error('Failed to increment view count');
            console.error(err);
          });
-          this.uids = this.db_file['uid'];
+          this.uids = [this.db_file['uid']];
          this.show_player = true;
          this.parseFileNames();
        }
@@ -304,12 +304,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
  }

  downloadContent() {
-    const fileNames = [];
-    for (let i = 0; i < this.playlist.length; i++) {
-      fileNames.push(this.playlist[i].title);
-    }
-
-    const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0];
+    const zipName = this.db_playlist.name;
    this.downloading = true;
    this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => {
      this.downloading = false;
--- a/src/app/posts.services.ts
+++ b/src/app/posts.services.ts
@@ -184,6 +184,18 @@ export class PostsService implements CanActivate {
                                                    cropFileSettings: cropFileSettings}, this.httpOptions);
    }

+    getDBInfo() {
+        return this.http.post(this.path + 'getDBInfo', {}, this.httpOptions);
+    }
+
+    transferDB(local_to_remote) {
+        return this.http.post(this.path + 'transferDB', {local_to_remote: local_to_remote}, this.httpOptions);
+    }
+
+    testConnectionString() {
+        return this.http.post(this.path + 'testConnectionString', {}, this.httpOptions);
+    }
+
    killAllDownloads() {
        return this.http.post(this.path + 'killAllDownloads', {}, this.httpOptions);
    }
@@ -236,11 +248,12 @@ export class PostsService implements CanActivate {
        return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions);
    }

-    downloadFileFromServer(uid, uuid = null, sub_id = null) {
+    downloadFileFromServer(uid, uuid = null, sub_id = null, is_playlist = null) {
        return this.http.post(this.path + 'downloadFileFromServer', {
                                                            uid: uid,
                                                            uuid: uuid,
-                                                            sub_id: sub_id
+                                                            sub_id: sub_id,
+                                                            is_playlist: is_playlist
                                                            },
                                                          {responseType: 'blob', params: this.httpOptions.params});
    }
--- a/src/app/settings/settings.component.html
+++ b/src/app/settings/settings.component.html
@@ -280,6 +280,43 @@
            </div>
        </ng-template>
    </mat-tab>
+    <!-- Database -->
+    <mat-tab label="Database" i18n-label="Database settings label">
+        <ng-template matTabContent>
+            <div *ngIf="new_config" class="container-fluid">
+                <div class="row">
+                    <div class="col-12 mt-3">
+                        <div *ngIf="db_info">
+                            <h5 i18n="Database info title">Database Info</h5>
+                            <p><ng-container i18n="Database location label">Database location:</ng-container>&nbsp;<strong>{{db_info['using_local_db'] ? 'Local' : 'MongoDB'}}</strong></p>
+                            <h6 i18n="Records per table label">Records per table</h6>
+                            <mat-list style="padding-top: 0px">
+                                <mat-list-item style="height: 28px" *ngFor="let table_stats of db_info['stats_by_table'] | keyvalue">
+                                    {{table_stats.key}}: {{table_stats.value.records_count}}
+                                </mat-list-item>
+                            </mat-list>
+
+                            <mat-form-field style="width: 100%; margin-top: 15px; margin-bottom: 10px" color="accent">
+                                <input [(ngModel)]="new_config['Database']['mongodb_connection_string']" matInput placeholder="MongoDB Connection String" i18n-placeholder="MongoDB Connection String" required>
+                                <mat-hint><ng-container i18n="MongoDB Connection String setting hint AKA preamble">Example:</ng-container>&nbsp;mongodb://127.0.0.1:27017/?compressors=zlib</mat-hint>
+                            </mat-form-field>
+
+                            <br>
+
+                            <button (click)="testConnectionString()" [disabled]="testing_connection_string" mat-flat-button color="accent"><ng-container i18n="Test connection string button">Test connection string</ng-container></button>
+
+                            <br>
+
+                            <button class="transfer-db-button" [disabled]="db_transferring" color="accent" (click)="transferDB()" mat-raised-button><ng-container i18n="Transfer DB button">Transfer DB to </ng-container>{{db_info['using_local_db'] ? 'MongoDB' : 'Local'}}</button>
+                        </div>
+                        <div *ngIf="!db_info">
+                            <ng-container i18n="Database info not retrieved error message">Database information could not be retrieved. Check the server logs for more information.</ng-container>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </ng-template>
+    </mat-tab>
    <!-- Advanced -->
    <mat-tab label="Advanced" i18n-label="Host settings label">
        <ng-template matTabContent>
--- a/src/app/settings/settings.component.scss
+++ b/src/app/settings/settings.component.scss
@@ -77,8 +77,13 @@
 }

 .category-custom-placeholder {
-background: #ccc;
-border: dotted 3px #999;
-min-height: 60px;
-transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
+    background: #ccc;
+    border: dotted 3px #999;
+    min-height: 60px;
+    transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
+}
+
+.transfer-db-button {
+    margin-top: 10px;
+    margin-bottom: 10px;
 }
--- a/src/app/settings/settings.component.ts
+++ b/src/app/settings/settings.component.ts
@@ -29,6 +29,10 @@ export class SettingsComponent implements OnInit {
  generated_bookmarklet_code = null;
  bookmarkletAudioOnly = false;

+  db_info = null;
+  db_transferring = false;
+  testing_connection_string = false;
+
  _settingsSame = true;

  latestGithubRelease = null;
@@ -48,6 +52,7 @@ export class SettingsComponent implements OnInit {

  ngOnInit() {
    this.getConfig();
+    this.getDBInfo();

    this.generated_bookmarklet_code = this.sanitizer.bypassSecurityTrustUrl(this.generateBookmarkletCode());

@@ -263,6 +268,60 @@ export class SettingsComponent implements OnInit {
    });
  }

+  getDBInfo() {
+    this.postsService.getDBInfo().subscribe(res => {
+      this.db_info = res['db_info'];
+    });
+  }
+
+  transferDB() {
+    const dialogRef = this.dialog.open(ConfirmDialogComponent, {
+      data: {
+        dialogTitle: 'Transfer DB',
+        dialogText: `Are you sure you want to transfer the DB?`,
+        submitText: 'Transfer',
+      }
+    });
+    dialogRef.afterClosed().subscribe(confirmed => {
+      if (confirmed) {
+        this._transferDB();
+      }
+    });
+  }
+
+  _transferDB() {
+    this.db_transferring = true;
+    this.postsService.transferDB(this.db_info['using_local_db']).subscribe(res => {
+      this.db_transferring = false;
+      const success = res['success'];
+      if (success) {
+        this.openSnackBar('Successfully transfered DB! Reloading info...');
+        this.getDBInfo();
+      } else {
+        this.openSnackBar('Failed to transfer DB -- transfer was aborted. Error: ' + res['error']);
+      }
+    }, err => {
+      this.db_transferring = false;
+      this.openSnackBar('Failed to transfer DB -- API call failed. See browser logs for details.');
+      console.error(err);
+    });
+  }
+
+  testConnectionString() {
+    this.testing_connection_string = true;
+    this.postsService.testConnectionString().subscribe(res => {
+      this.testing_connection_string = false;
+      if (res['success']) {
+        this.postsService.openSnackBar('Connection successful!');
+      } else {
+        this.postsService.openSnackBar('Connection failed! Error: ' + res['error']);
+      }
+    }, err => {
+      this.testing_connection_string = false;
+      this.postsService.openSnackBar('Connection failed! Error: Server error. See logs for more info.');
+    });
+  }
+
  // snackbar helper
  public openSnackBar(message: string, action: string = '') {
    this.snackBar.open(message, action, {