Archive improvements

Began UI for viewing/modifying archives

Public API v1.yaml

@@ -578,6 +578,69 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
+  /api/deleteArchiveItem:
+    post:
+      tags:
+        - archive
+      summary: Delete item from archive
+      description: 'Deletes an item from the archive'
+      operationId: post-api-deleteArchiveItem
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DeleteArchiveItemRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/importArchive:
+    post:
+      tags:
+        - archive
+      summary: Imports archive
+      description: 'Imports an existing archive.txt file'
+      operationId: post-api-importArchive
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/ImportArchiveRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/uploadCookies:
+    post:
+      tags:
+        - downloader
+      summary: Upload cookies
+      description: 'Uploads cookies file to be used during downloading'
+      operationId: post-api-uploadCookies
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadCookiesRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -2089,6 +2152,84 @@ components:
           $ref: '#/components/schemas/FileType'
         sub_id:
           type: string
+    Archive:
+      required:
+        - extractor
+        - id
+        - type
+        - title
+        - timestamp
+        - uid
+      type: object
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        title:
+          type: string
+        user_uid:
+          type: string
+        sub_id:
+          type: string
+        timestamp:
+          type: number
+        uid:
+          type: string
+    DeleteArchiveItemRequest:
+      type: object
+      required:
+        - extractor
+        - id
+        - type
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    ImportArchiveRequest:
+      type: object
+      required:
+        - archive
+        - type
+      properties:
+        archive:
+          type: string
+          format: binary
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesRequest:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesResponse:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    UploadCookiesRequest:
+      type: object
+      required:
+        - cookies
+      properties:
+        cookies:
+          type: string
+          format: binary
     UpdaterStatus:
       required:
         - details

backend/archive.js

@@ -1,5 +1,6 @@
 const path = require('path');
 const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
 
 const db_api = require('./db');
 
@@ -11,8 +12,8 @@ exports.generateArchive = async (type = null, user_uid = null, sub_id = null) =>
     return archive_item_lines.join('\n');
 }
 
-exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
-    const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
+    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
     const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
     return success;
 }
@@ -43,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id =
 
         // we can't do a bulk write because we need to avoid duplicate archive items existing in db
 
-        const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
+        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
         await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id});
         archive_import_count++;
     }
@@ -76,12 +77,15 @@ exports.importArchives = async () => {
     return imported_archives;
 }
 
-const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => {
+const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
     return {
         extractor: extractor,
         id: id,
         type: type,
+        title: title,
         user_uid: user_uid ? user_uid : null,
-        sub_id: sub_id ? sub_id : null
+        sub_id: sub_id ? sub_id : null,
+        timestamp: Date.now() / 1000,
+        uid: uuid()
     }
 }

backend/downloader.js

@@ -369,7 +369,7 @@ async function downloadQueuedFile(download_uid) {
                     const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
                     const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']);
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
 
                     notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
 

src/api-types/index.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
 export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
@@ -26,6 +27,7 @@ export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
 export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteArchiveItemRequest } from './models/DeleteArchiveItemRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
 export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
@@ -51,6 +53,8 @@ export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetArchivesRequest } from './models/GetArchivesRequest';
+export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -75,6 +79,7 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
@@ -117,6 +122,7 @@ export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
 export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/Archive.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type Archive = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    title: string;
+    user_uid?: string;
+    sub_id?: string;
+    timestamp: number;
+    uid: string;
+};

src/api-types/models/DeleteArchiveItemRequest.ts

@@ -0,0 +1,12 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type DeleteArchiveItemRequest = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    sub_id?: string;
+};

src/api-types/models/GetArchivesRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type GetArchivesRequest = {
+    type?: FileType;
+    sub_id?: string;
+};

src/api-types/models/GetArchivesResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type GetArchivesResponse = {
+    archives: Array<Archive>;
+};

src/api-types/models/ImportArchiveRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type ImportArchiveRequest = {
+    archive: Blob;
+    type: FileType;
+    sub_id?: string;
+};

src/api-types/models/UploadCookiesRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UploadCookiesRequest = {
+    cookies: Blob;
+};

src/app/app.module.ts

@@ -94,6 +94,7 @@ import { TaskSettingsComponent } from './components/task-settings/task-settings.
 import { GenerateRssUrlComponent } from './dialogs/generate-rss-url/generate-rss-url.component';
 import { SortPropertyComponent } from './components/sort-property/sort-property.component';
 import { OnlyNumberDirective } from './directives/only-number.directive';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 registerLocaleData(es, 'es');
 
@@ -145,7 +146,8 @@ registerLocaleData(es, 'es');
         TaskSettingsComponent,
         GenerateRssUrlComponent,
         SortPropertyComponent,
-        OnlyNumberDirective
+        OnlyNumberDirective,
+        ArchiveViewerComponent
     ],
     imports: [
         CommonModule,

src/app/components/archive-viewer/archive-viewer.component.html

@@ -0,0 +1,51 @@
+<!-- (selectionChange)="sidePanelModeChanged($event.value)" -->
+
+<div [hidden]="!(archives && archives.length > 0)">
+    <div style="overflow: hidden;" class="mat-elevation-z8">
+        <mat-table style="overflow: hidden" matSort [dataSource]="dataSource">
+      
+          <!-- Date Column -->
+          <ng-container matColumnDef="timestamp">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Date">Date</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element"> {{element.timestamp | date: 'short'}} </mat-cell>
+          </ng-container>
+      
+          <!-- Title Column -->
+          <ng-container matColumnDef="title">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.title ? element.title : null">
+                    {{element.title}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+           <!-- ID Column -->
+           <ng-container matColumnDef="id">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="ID">ID</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.title ? element.title : null">
+                    {{element.id}}
+                </span>
+            </mat-cell>
+          </ng-container>
+
+          <!-- Extractor Column -->
+          <ng-container matColumnDef="extractor">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Extractor">Extractor</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.extractor? element.extractor : null">
+                    {{element.extractor}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+            <mat-header-row *matHeaderRowDef="displayedColumns"></mat-header-row>
+            <mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
+        </mat-table>
+      </div>
+</div>
+
+<div *ngIf="(!archives || archives.length === 0)">
+  <h4 style="text-align: center; margin-top: 10px;" i18n="Archives empty">Archives empty</h4>
+</div>

src/app/components/archive-viewer/archive-viewer.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ArchiveViewerComponent } from './archive-viewer.component';
+
+describe('ArchiveViewerComponent', () => {
+  let component: ArchiveViewerComponent;
+  let fixture: ComponentFixture<ArchiveViewerComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ArchiveViewerComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(ArchiveViewerComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/archive-viewer/archive-viewer.component.ts

@@ -0,0 +1,41 @@
+import { Component, ViewChild } from '@angular/core';
+import { MatSort } from '@angular/material/sort';
+import { MatTableDataSource } from '@angular/material/table';
+import { Archive } from 'api-types/models/Archive';
+import { PostsService } from 'app/posts.services';
+
+@Component({
+  selector: 'app-archive-viewer',
+  templateUrl: './archive-viewer.component.html',
+  styleUrls: ['./archive-viewer.component.scss']
+})
+export class ArchiveViewerComponent {
+  archives = null;
+  displayedColumns: string[] = ['timestamp', 'title', 'id', 'extractor'];
+  dataSource = null;
+  archives_retrieved = false;
+
+  @ViewChild(MatSort) sort: MatSort;
+  
+  constructor(private postsService: PostsService) {
+
+  }
+
+  filterSelectionChanged(value: string): void {
+    this.getArchives(value);
+  }
+
+  getArchives(sub_id: string = null): void {
+    this.postsService.getArchives(sub_id).subscribe(res => {
+      if (res['archives'] !== null 
+        && res['archives'] !== undefined
+        && JSON.stringify(this.archives) !== JSON.stringify(res['archives'])) {
+          this.archives = res['archives']
+          this.dataSource = new MatTableDataSource<Archive>(this.archives);
+          this.dataSource.sort = this.sort;
+      } else {
+        // failed to get downloads
+      }
+    });
+  }
+}

src/app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component.ts

@@ -32,10 +32,7 @@ export class CookiesUploaderDialogComponent implements OnInit {
       if (droppedFile.fileEntry.isFile) {
         const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
         fileEntry.file((file: File) => {          
-          // You could upload it like this:
+          this.postsService.uploadCookiesFile(file, droppedFile.relativePath).subscribe(res => {
-          const formData = new FormData()
-          formData.append('cookies', file, droppedFile.relativePath);
-          this.postsService.uploadCookiesFile(formData).subscribe(res => {
             this.uploading = false;
             if (res['success']) {
               this.uploaded = true;

src/app/posts.services.ts

@@ -106,7 +106,10 @@ import {
     SetNotificationsToReadRequest,
     GetNotificationsResponse,
     UpdateTaskOptionsRequest,
-    User
+    User,
+    DeleteArchiveItemRequest,
+    GetArchivesRequest,
+    GetArchivesResponse
 } from '../api-types';
 import { isoLangs } from './settings/locales_list';
 import { Title } from '@angular/platform-browser';
@@ -431,8 +434,11 @@ export class PostsService implements CanActivate {
         return this.http.post<UpdateConcurrentStreamResponse>(this.path + 'updateConcurrentStream', body, this.httpOptions);
     }
 
-    uploadCookiesFile(fileFormData) {
+    uploadCookiesFile(cookiesFile: File, cookiesFilePath: string) {
-        return this.http.post<SuccessObject>(this.path + 'uploadCookies', fileFormData, this.httpOptions);
+        const formData = new FormData()
+        formData.append('cookies', cookiesFile, cookiesFilePath);
+        // const body: UploadCookiesRequest = formData;
+        return this.http.post<SuccessObject>(this.path + 'uploadCookies', formData, this.httpOptions);
     }
 
     downloadArchive(type: FileType, sub_id: string) {
@@ -440,6 +446,24 @@ export class PostsService implements CanActivate {
         return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params});
     }
 
+    getArchives(sub_id: string) {
+        const body: GetArchivesRequest = {sub_id: sub_id};
+        return this.http.post<GetArchivesResponse>(this.path + 'getArchives', body, this.httpOptions);
+    }
+
+    importArchive(archiveFile: File, type: FileType, sub_id: string = null) {
+        const formData = new FormData()
+        formData.append('archive', archiveFile, 'archive.txt');
+        formData.append('type', type);
+        formData.append('sub_id', sub_id);
+        return this.http.post<SuccessObject>(this.path + 'importArchive', formData, this.httpOptions);
+    }
+
+    deleteArchiveItem(extractor: string, id: string, type: FileType, sub_id: string = null) {
+        const body: DeleteArchiveItemRequest = {extractor: extractor, id: id, type: type, sub_id: sub_id};
+        return this.http.post<SuccessObject>(this.path + 'deleteArchiveItem', body, this.httpOptions);
+    }
+
     getFileFormats(url) {
         const body: GetFileFormatsRequest = {url: url};
         return this.http.post<GetFileFormatsResponse>(this.path + 'getFileFormats', body, this.httpOptions);