Merge branch 'gh-pages' of https://github.com/Tzahi12345/YoutubeDL-Material into gh-pages

Pre 4.3.1 bug fixes

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
       - name: setup node
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
           cache: 'npm'
       - name: install dependencies
         run: |

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+    "recommendations": [
+        "angular.ng-template",
+        "dbaeumer.vscode-eslint",
+        "waderyan.gitblame",
+        "42crunch.vscode-openapi",
+        "redhat.vscode-yaml",
+        "christian-kohler.npm-intellisense",
+        "hbenl.vscode-mocha-test-adapter"
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "mochaExplorer.files": "backend/test/**/*.js",
+    "mochaExplorer.cwd": "backend",
+    "mochaExplorer.globImplementation": "vscode",
+    "mochaExplorer.env": {
+        "YTDL_MODE": "debug"
+    }
+}

.vscode/tasks.json

@@ -1,25 +1,60 @@
 {
 	"version": "2.0.0",
+	"windows": {
+        "options": {
+            "shell": {
+                "executable": "cmd.exe",
+                "args": [
+                    "/d", "/c"
+                ]
+            }
+        }
+    },
 	"tasks": [
 		{
 			"type": "npm",
 			"script": "start",
 			"problemMatcher": [],
 			"label": "Dev: start frontend",
-			"detail": "ng serve"
+			"detail": "ng serve",
+			"presentation": {
+				"echo": true,
+				"reveal": "always",
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
+			}
 		},
 		{
 			"label": "Dev: start backend",
 			"type": "shell",
-			"command": "set YTDL_MODE=debug && node app.js",
+			"command": "node app.js",
 			"options": {
-				"cwd": "./backend"
+				"cwd": "./backend",
+				"env": {
+					"YTDL_MODE": "debug"
+				}
 			},
 			"presentation": {
+				"echo": true,
 				"reveal": "always",
-				"panel": "new"
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
 			},
-			"problemMatcher": []
+			"problemMatcher": [],
+			"dependsOn": ["Dev: post-build"]
+		},
+		{
+			"label": "Dev: post-build",
+			"type": "shell",
+			"command": "node src/postbuild.mjs"
+		},
+		{
+			"label": "Dev: run all",
+			"dependsOn": ["Dev: start backend", "Dev: start frontend"]
 		}
 	]
 }

Dockerfile

@@ -2,11 +2,12 @@
 FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY ffmpeg-fetch.sh .
+COPY docker-utils/ffmpeg-fetch.sh .
+RUN chmod +x ffmpeg-fetch.sh
 RUN sh ./ffmpeg-fetch.sh
 
 
-# Create our Ubuntu 22.04 with node 16
+# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
 # Go to 20.04
 FROM ubuntu:20.04 AS base
 ARG DEBIAN_FRONTEND=noninteractive
@@ -21,7 +22,8 @@ RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
     apt install -y --no-install-recommends curl ca-certificates tzdata && \
     curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
     apt install -y --no-install-recommends nodejs && \
-    npm -g install npm && \
+    npm -g install npm n && \
+    n 16.14.2 && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -45,21 +47,31 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
 RUN npm install -g pm2 && \
     apt update && \
-    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
-RUN pip install tcd
+RUN pip install pycryptodomex
 WORKDIR /app
 # User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
 COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
+RUN chown $UID:$GID .
 RUN chmod +x /app/fix-scripts/*.sh
 # Add some persistence data
 #VOLUME ["/app/appdata"]

Public API v1.yaml

@@ -111,6 +111,37 @@ paths:
                 $ref: '#/components/schemas/GetAllFilesResponse'
       security:
         - Auth query parameter: []
+  /api/rss:
+    get:
+      tags:
+        - files
+      summary: Generates an RSS feed
+      description: Generates an RSS feed for downloaded files
+      operationId: get-rss
+      parameters:
+        - in: query
+          name: params
+          schema:
+            allOf:
+            - $ref: '#/components/schemas/GetAllFilesRequest'
+            - type: object
+              properties:
+                uuid:
+                  type: string
+                  description: user uid
+                  default: null
+          style: form
+          explode: true
+      responses:
+        '200':
+          description: OK
+          content:
+            text/plain:
+              schema:
+                type: string
+                description: RSS feed
+      security:
+        - Auth query parameter: []
   /api/getFile:
     post:
       tags:
@@ -547,6 +578,69 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
+  /api/deleteArchiveItems:
+    post:
+      tags:
+        - archive
+      summary: Delete item from archive
+      description: 'Deletes an item from the archive'
+      operationId: post-api-deleteArchiveItems
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DeleteArchiveItemsRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/importArchive:
+    post:
+      tags:
+        - archive
+      summary: Imports archive
+      description: 'Imports an existing archive.txt file'
+      operationId: post-api-importArchive
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ImportArchiveRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/uploadCookies:
+    post:
+      tags:
+        - downloader
+      summary: Upload cookies
+      description: 'Uploads cookies file to be used during downloading'
+      operationId: post-api-uploadCookies
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadCookiesRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -811,7 +905,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/SuccessObject'
+                $ref: '#/components/schemas/RestartDownloadResponse'
       requestBody:
         content:
           application/json:
@@ -1589,6 +1683,10 @@ components:
           type: string
           description: Height of the video, if known
           example: '1080'
+        maxHeight:
+          type: string
+          description: Max height that should be used, useful for playlists. selectedHeight will override this.
+          example: '1080'
         maxBitrate:
           type: string
           description: Specify ffmpeg/avconv audio quality
@@ -1597,6 +1695,9 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
+        ignoreArchive:
+          type: boolean
+          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -1621,6 +1722,13 @@ components:
       properties:
         download:
           $ref: '#/components/schemas/Download'
+    RestartDownloadResponse:
+      allOf:
+        - $ref: '#/components/schemas/SuccessObject'
+        - type: object
+          properties:
+            new_download_uid:
+              type: string
     GetAllDownloadsRequest:
       type: object
       properties:
@@ -1673,6 +1781,16 @@ components:
       required:
         - task_key
         - new_data
+    UpdateTaskOptionsRequest:
+      type: object
+      properties:
+        task_key:
+          type: string
+        new_options:
+          type: object
+      required:
+        - task_key
+        - new_options
     GetTaskResponse:
       type: object
       properties:
@@ -1741,29 +1859,39 @@ components:
             description: Two elements allowed, start index and end index
           minItems: 2
           maxItems: 2
+          default: null
         text_search:
           type: string
           description: Filter files by title
+          default: null
         file_type_filter:
           $ref: '#/components/schemas/FileTypeFilter'
+        favorite_filter:
+          type: boolean
+          description: If set to true, only gets favorites
+          default: false
         sub_id:
           type: string
           description: Include if you want to filter by subscription
+          default: null
     Sort:
       type: object
       properties:
         by:
           type: string
           description: Property to sort by
+          default: registered
         order:
           type: number
           description: 1 for ascending, -1 for descending
+          default: -1
     FileTypeFilter:
       type: string
       enum:
         - audio_only
         - video_only
         - both
+      default: both
     GetAllFilesResponse:
       required:
         - files
@@ -1881,16 +2009,11 @@ components:
           description: Number of files removed
     DeleteSubscriptionFileRequest:
       required:
-        - file
-        - sub
+        - file_uid
       type: object
       properties:
-        file:
-          type: string
         file_uid:
           type: string
-        sub:
-          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteForever:
           type: boolean
           description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -2030,17 +2153,83 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      required:
-        - sub
       type: object
       properties:
-        sub:
-          required:
-            - archive_dir
-          type: object
-          properties:
-            archive_dir:
-              type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    Archive:
+      required:
+        - extractor
+        - id
+        - type
+        - title
+        - timestamp
+        - uid
+      type: object
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        title:
+          type: string
+        user_uid:
+          type: string
+        sub_id:
+          type: string
+        timestamp:
+          type: number
+        uid:
+          type: string
+    DeleteArchiveItemsRequest:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    ImportArchiveRequest:
+      type: object
+      required:
+        - archive
+        - type
+      properties:
+        archive:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesRequest:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesResponse:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    UploadCookiesRequest:
+      type: object
+      required:
+        - cookies
+      properties:
+        cookies:
+          type: string
+          format: binary
     UpdaterStatus:
       required:
         - details
@@ -2061,8 +2250,6 @@ components:
         tag:
           type: string
     DBInfoResponse:
-      required:
-        - db_info
       type: object
       properties:
         using_local_db:
@@ -2084,6 +2271,8 @@ components:
               $ref: '#/components/schemas/TableInfo'
             download_queue:
               $ref: '#/components/schemas/TableInfo'
+            archives:
+              $ref: '#/components/schemas/TableInfo'
     TransferDBResponse:
       required:
         - success
@@ -2383,6 +2572,7 @@ components:
         - upload_date
         - uploader
         - url
+        - favorite
       type: object
       properties:
         id:
@@ -2412,6 +2602,8 @@ components:
           type: string
         uid:
           type: string
+        user_uid:
+          type: string
         sharingEnabled:
           type: boolean
         category:
@@ -2430,6 +2622,8 @@ components:
         abr:
           type: number
           description: In Kbps
+        favorite:
+          type: boolean
     Playlist:
       required:
         - uids
@@ -2461,6 +2655,8 @@ components:
           type: string
         auto:
           type: boolean
+        sharingEnabled:
+          type: boolean
     Download:
       required:
         - url
@@ -2505,6 +2701,10 @@ components:
           type: string
           description: Error text, set if download fails.
           nullable: true
+        error_type:
+          type: string
+          description: Error type, may or may not be set in case of an error
+          nullable: true
         user_uid:
           type: string
         sub_id:
@@ -2543,6 +2743,8 @@ components:
           type: string
         schedule:
           type: object
+        options:
+          type: object
     Schedule:
       required:
         - type
@@ -2567,6 +2769,8 @@ components:
               type: number
             timestamp:
               type: number
+            tz:
+              type: string
     DBBackup:
       required:
         - name
@@ -2755,6 +2959,44 @@ components:
           type: string
         date:
           type: string
+    Notification:
+      required:
+        - uid
+        - type
+        - text
+        - read
+        - timestamp
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/NotificationType'
+        uid:
+          type: string
+        user_uid:
+          type: string
+        action:
+          type: array
+          items:
+            $ref: '#/components/schemas/NotificationAction'
+        read:
+          type: boolean
+        data:
+          type: object
+        timestamp:
+          type: number
+    NotificationAction:
+      type: string
+      enum:
+      - play
+      - retry_download
+      - view_download_error
+      - view_tasks
+    NotificationType:
+      type: string
+      enum:
+      - download_complete
+      - download_error
+      - task_finished
     BaseChangePermissionsRequest:
       required:
         - permission
@@ -2886,6 +3128,29 @@ components:
                   type: array
                   items:
                     $ref: '#/components/schemas/UserPermission'
+    DeleteNotificationRequest:
+      required:
+        - uid
+      type: object
+      properties:
+        uid:
+          type: string
+    SetNotificationsToReadRequest:
+      required:
+        - uids
+      type: object
+      properties:
+        uids:
+          type: array
+          items:
+            type: string
+    GetNotificationsResponse:
+      type: object
+      properties:
+        notifications:
+          type: array
+          items:
+            $ref: '#/components/schemas/Notification'
   securitySchemes:
     Auth query parameter:
       name: apiKey

README.md

@@ -6,7 +6,7 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
@@ -14,7 +14,7 @@ Now with [Docker](#Docker) support!
 
 ## Getting Started
 
-Check out the prerequisites, and go to the installation section. Easy as pie!
+Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
@@ -52,6 +52,8 @@ Optional dependencies:
 
 ### Installing
 
+If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
+
 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.

angular.json

@@ -182,7 +182,6 @@
       }
     }
   },
-  "defaultProject": "youtube-dl-material",
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",
@@ -191,5 +190,8 @@
     "@schematics/angular:directive": {
       "prefix": "app"
     }
+  },
+  "cli": {
+    "analytics": false
   }
 }

backend/app.js

@@ -7,7 +7,6 @@ const path = require('path');
 const compression = require('compression');
 const multer  = require('multer');
 const express = require("express");
-const session = require("express-session");
 const bodyParser = require("body-parser");
 const archiver = require('archiver');
 const unzipper = require('unzipper');
@@ -19,6 +18,7 @@ const URL = require('url').URL;
 const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
+const Feed = require('feed').Feed;
 
 // needed if bin/details somehow gets deleted
 if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -33,6 +33,7 @@ const subscriptions_api = require('./subscriptions');
 const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 var app = express();
 
@@ -70,7 +71,8 @@ db.defaults(
         downloads: {},
         subscriptions: [],
         files_to_db_migration_complete: false,
-        tasks_manager_role_migration_complete: false
+        tasks_manager_role_migration_complete: false,
+        archives_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -159,11 +161,6 @@ app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
 // use passport
-app.use(session({
-    resave: false,
-    saveUninitialized: true,
-    secret: 'ytdl-material-secret' 
-}));
 app.use(auth_api.passport.initialize());
 app.use(auth_api.passport.session());
 
@@ -206,6 +203,15 @@ async function checkMigrations() {
         db.set('tasks_manager_role_migration_complete', true).write();
     }
 
+    const archives_migration_complete = db.get('archives_migration_complete').value();
+    if (!archives_migration_complete) {
+        logger.info('Checking if archives have been migrated...');
+        const imported_archives = await archive_api.importArchives();
+        if (imported_archives) logger.info('Archives migration complete!');
+        else logger.error('Failed to migrate archives!');
+        db.set('archives_migration_complete', true).write();
+    }
+
     return true;
 }
 
@@ -517,9 +523,6 @@ async function loadConfig() {
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
-    // creates archive path if missing
-    await fs.ensureDir(utils.getArchiveFolder());
-
     // check migrations
     await checkMigrations();
 
@@ -565,14 +568,7 @@ function loadConfigValues() {
     url_domain = new URL(url);
 
     let logger_level = config_api.getConfigItem('ytdl_logger_level');
-    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
-    if (!possible_levels.includes(logger_level)) {
-        logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
-        logger_level = 'info';
-    }
-    logger.level = logger_level;
-    winston.loggers.get('console').level = logger_level;
-    logger.transports[2].level = logger_level;
+    utils.updateLoggerLevel(logger_level);
 }
 
 function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -707,7 +703,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -780,7 +776,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
 
 app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
     const db_info = await db_api.getDBStats();
-    res.send({db_info: db_info});
+    res.send(db_info);
 });
 
 app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -820,11 +816,13 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings
+        cropFileSettings: req.body.cropFileSettings,
+        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -850,6 +848,7 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
@@ -928,35 +927,15 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
-    let files = null;
     const sort = req.body.sort;
     const range = req.body.range;
     const text_search = req.body.text_search;
     const file_type_filter = req.body.file_type_filter;
+    const favorite_filter = req.body.favorite_filter;
     const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const filter_obj = {user_uid: uuid};
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (sub_id) {
-        filter_obj['sub_id'] = sub_id;
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
-    const file_count = await db_api.getRecords('files', filter_obj, true);
-
-    files = JSON.parse(JSON.stringify(files));
+    const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
 
     res.send({
         files: files,
@@ -1099,9 +1078,6 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
             await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
         } else if (is_playlist) {
             await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
-        } else if (type === 'subscription') {
-            // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
-            //          time they are requested from the subscription directory.
         } else {
             // error
             success = false;
@@ -1116,7 +1092,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
     });
 });
 
-app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
+app.post('/api/incrementViewCount', async (req, res) => {
     let file_uid = req.body.file_uid;
     let sub_id = req.body.sub_id;
     let uuid = req.body.uuid;
@@ -1251,12 +1227,9 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
-    let file = req.body.file;
     let file_uid = req.body.file_uid;
-    let sub = req.body.sub;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
+    let success = await db_api.deleteFile(file_uid, deleteForever);
 
     if (success) {
         res.send({
@@ -1437,10 +1410,9 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
 app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const uid = req.body.uid;
     const blacklistMode = req.body.blacklistMode;
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     let wasDeleted = false;
-    wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
+    wasDeleted = await db_api.deleteFile(uid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1472,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
+        wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1533,20 +1505,69 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     });
 });
 
+app.post('/api/getArchives', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id;
+    const filter_obj = {user_uid: uuid, sub_id: sub_id};
+    const type = req.body.type;
+
+    // we do this for file types because if type is null, that means get files of all types
+    if (type) filter_obj['type'] = type;
+
+    const archives = await db_api.getRecords('archives', filter_obj);
+
+    res.send({
+        archives: archives
+    });
+});
+
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    let sub = req.body.sub;
-    let archive_dir = sub.archive;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
 
-    let full_archive_path = path.join(archive_dir, 'archive.txt');
+    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
 
-    if (await fs.pathExists(full_archive_path)) {
-        res.sendFile(full_archive_path);
+    if (archive_text !== null && archive_text !== undefined) {
+        res.setHeader('Content-type', "application/octet-stream");
+        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
+        res.send(archive_text);
     } else {
-        res.sendStatus(404);
+        res.sendStatus(400);
     }
 
 });
 
+app.post('/api/importArchive', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archive = req.body.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
+
+    const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
+
+    const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
+
+    res.send({
+        success: !!imported_count,
+        imported_count: imported_count
+    });
+});
+
+app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archives = req.body.archives;
+
+    let success = true;
+    for (const archive of archives) {
+        success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
+    }
+
+    res.send({
+        success: success
+    });
+});
+
 var upload_multer = multer({ dest: __dirname + '/appdata/' });
 app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
     const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1618,7 +1639,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
-        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
+        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1738,8 +1759,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
 
 app.post('/api/restartDownload', optionalJwt, async (req, res) => {
     const download_uid = req.body.download_uid;
-    const success = await downloader_api.restartDownload(download_uid);
-    res.send({success: success});
+    const new_download = await downloader_api.restartDownload(download_uid);
+    res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
 });
 
 app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1816,6 +1837,15 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
+    const task_key = req.body.task_key;
+    const new_options = req.body.new_options;
+  
+    const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
+
+    res.send({success: success});
+});
+
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
     fs.ensureDirSync(backup_dir);
@@ -1999,6 +2029,93 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+// notifications
+
+app.post('/api/getNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
+
+    res.send({notifications: notifications});
+});
+
+// set notifications to read
+app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
+    const uid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeRecord('notifications', {uid: uid});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
+
+    res.send({success: success});
+});
+
+// rss feed
+
+app.get('/api/rss', async function (req, res) {
+    if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
+        logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
+        res.sendStatus(403);
+        return;
+    }
+
+    // these are returned
+    const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
+    const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
+    const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
+    const file_type_filter = req.query.file_type_filter;
+    const favorite_filter = req.query.favorite_filter === 'true';
+    const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
+    const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
+
+    const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+
+    const feed = new Feed({
+            title: 'Downloads',
+            description: 'YoutubeDL-Material downloads',
+            id: utils.getBaseURL(),
+            link: utils.getBaseURL(),
+            image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
+            favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
+            generator: 'YoutubeDL-Material'
+    });
+
+    files.forEach(file => {
+        feed.addItem({
+            title: file.title,
+            link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
+            description: file.description,
+            author: [
+                {
+                    name: file.uploader,
+                    link: file.url
+                }
+            ],
+            contributor: [],
+            date: file.timestamp,
+            // https://stackoverflow.com/a/45415677/8088021
+            image: file.thumbnailURL.replace('&', '&amp;')
+        });
+      });
+    res.send(feed.rss2());
+});
+
+// web server
+
 app.use(function(req, res, next) {
     //if the request is not html then move along
     var accept = req.accepts('html', 'json', 'xml');

backend/appdata/default.json

@@ -23,7 +23,12 @@
       "download_only_mode": false,
       "allow_autoplay": true,
       "enable_downloads_manager": true,
-      "allow_playlist_categorization": true
+      "allow_playlist_categorization": true,
+      "force_autoplay": false,
+      "enable_notifications": true,
+      "enable_all_notifications": true,
+      "allowed_notification_types": [],
+      "enable_rss_feed": false
     },
     "API": {
       "use_API_key": false,
@@ -35,7 +40,17 @@
       "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
-      "generate_NFO_files": false
+      "generate_NFO_files": false,
+      "use_ntfy_API": false,
+      "ntfy_topic_URL": "",
+      "use_gotify_API": false,
+      "gotify_server_URL": "",
+      "gotify_app_token": "",
+      "use_telegram_API": false,
+      "telegram_bot_token": "",
+      "telegram_chat_id": "",
+      "webhook_URL": "",
+      "discord_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",

backend/archive.js

@@ -0,0 +1,91 @@
+const path = require('path');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+
+const db_api = require('./db');
+
+exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
+    const filter = {user_uid: user_uid, sub_id: sub_id};
+    if (type) filter['type'] = type;
+    const archive_items = await db_api.getRecords('archives', filter);
+    const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
+    return archive_item_lines.join('\n');
+}
+
+exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
+    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
+    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
+    return success;
+}
+
+exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return success;
+}
+
+exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
+    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return !!archive_item;
+}
+
+exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
+    let archive_import_count = 0;
+    const lines = archive_text.split('\n');
+    for (let line of lines) {
+        const archive_line_parts = line.trim().split(' ');
+        // should just be the extractor and the video ID
+        if (archive_line_parts.length !== 2) {
+            continue;
+        }
+
+        const extractor = archive_line_parts[0];
+        const id        = archive_line_parts[1];
+        if (!extractor || !id) continue;
+
+        // we can't do a bulk write because we need to avoid duplicate archive items existing in db
+
+        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
+        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
+        archive_import_count++;
+    }
+    return archive_import_count;
+}
+
+exports.importArchives = async () => {
+    const imported_archives = [];
+    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        if (!dir_to_check['archive_path']) continue;
+
+        const files_to_import = [
+            path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
+            path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
+        ]
+
+        for (const file_to_import of files_to_import) {
+            const file_exists = await fs.pathExists(file_to_import);
+            if (!file_exists) continue;
+
+            const archive_text = await fs.readFile(file_to_import, 'utf8');
+            await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
+            imported_archives.push(file_to_import);
+        }
+    }
+    return imported_archives;
+}
+
+const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
+    return {
+        extractor: extractor,
+        id: id,
+        type: type,
+        title: title,
+        user_uid: user_uid ? user_uid : null,
+        sub_id: sub_id ? sub_id : null,
+        timestamp: Date.now() / 1000,
+        uid: uuid()
+    }
+}

backend/authentication/auth.js

@@ -33,7 +33,14 @@ exports.initialize = function () {
 
   saltRounds = 10;
 
+  // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
+  if (!(+JWT_EXPIRATION)) {
+    logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
+    JWT_EXPIRATION = 86400;
+  } else {
+    JWT_EXPIRATION = +JWT_EXPIRATION;
+  }
 
   SERVER_SECRET = null;
   if (db_api.users_db.get('jwt_secret').value()) {

backend/config.js

@@ -185,7 +185,6 @@ const DEFAULT_CONFIG = {
         "default_file_output": "",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false,
         "include_thumbnail": true,
         "include_metadata": true,
         "max_concurrent_downloads": 5,
@@ -196,21 +195,32 @@ const DEFAULT_CONFIG = {
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_autoplay": true,
+        "force_autoplay": false,
         "enable_downloads_manager": true,
-        "allow_playlist_categorization": true
+        "allow_playlist_categorization": true,
+        "enable_notifications": true,
+        "enable_all_notifications": true,
+        "allowed_notification_types": [],
+        "enable_rss_feed": false,
       },
       "API": {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
-        "use_twitch_API": false,
-        "twitch_client_ID": "",
-        "twitch_client_secret": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
-        "generate_NFO_files": false
+        "generate_NFO_files": false,
+        "use_ntfy_API": false,
+        "ntfy_topic_URL": "",
+        "use_gotify_API": false,
+        "gotify_server_URL": "",
+        "gotify_app_token": "",
+        "use_telegram_API": false,
+        "telegram_bot_token": "",
+        "telegram_chat_id": "",
+        "webhook_URL": "",
+        "discord_webhook_URL": ""
       },
       "Themes": {
         "default_theme": "default",

backend/consts.js

@@ -30,10 +30,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'
     },
-    'ytdl_safe_download_override': {
-        'key': 'ytdl_safe_download_override',
-        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
-    },
     'ytdl_include_thumbnail': {
         'key': 'ytdl_include_thumbnail',
         'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -68,9 +64,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_autoplay': {
-        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+    'ytdl_force_autoplay': {
+        'key': 'ytdl_force_autoplay',
+        'path': 'YoutubeDLMaterial.Extra.force_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -80,6 +76,22 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_allow_playlist_categorization',
         'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
+    'ytdl_enable_notifications': {
+        'key': 'ytdl_enable_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_notifications'
+    },
+    'ytdl_enable_all_notifications': {
+        'key': 'ytdl_enable_all_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
+    },
+    'ytdl_allowed_notification_types': {
+        'key': 'ytdl_allowed_notification_types',
+        'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
+    },
+    'ytdl_enable_rss_feed': {
+        'key': 'ytdl_enable_rss_feed',
+        'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
+    },
 
     // API
     'ytdl_use_api_key': {
@@ -98,18 +110,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
-    'ytdl_use_twitch_api': {
-        'key': 'ytdl_use_twitch_api',
-        'path': 'YoutubeDLMaterial.API.use_twitch_API'
-    },
-    'ytdl_twitch_client_id': {
-        'key': 'ytdl_twitch_client_id',
-        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
-    },
-    'ytdl_twitch_client_secret': {
-        'key': 'ytdl_twitch_client_secret',
-        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
-    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -122,6 +122,46 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_generate_nfo_files',
         'path': 'YoutubeDLMaterial.API.generate_NFO_files'
     },
+    'ytdl_use_ntfy_API': {
+        'key': 'ytdl_use_ntfy_API',
+        'path': 'YoutubeDLMaterial.API.use_ntfy_API'
+    },
+    'ytdl_ntfy_topic_url': {
+        'key': 'ytdl_ntfy_topic_url',
+        'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
+    },
+    'ytdl_use_gotify_API': {
+        'key': 'ytdl_use_gotify_API',
+        'path': 'YoutubeDLMaterial.API.use_gotify_API'
+    },
+    'ytdl_gotify_server_url': {
+        'key': 'ytdl_gotify_server_url',
+        'path': 'YoutubeDLMaterial.API.gotify_server_URL'
+    },
+    'ytdl_gotify_app_token': {
+        'key': 'ytdl_gotify_app_token',
+        'path': 'YoutubeDLMaterial.API.gotify_app_token'
+    },
+    'ytdl_use_telegram_API': {
+        'key': 'ytdl_use_telegram_API',
+        'path': 'YoutubeDLMaterial.API.use_telegram_API'
+    },
+    'ytdl_telegram_bot_token': {
+        'key': 'ytdl_telegram_bot_token',
+        'path': 'YoutubeDLMaterial.API.telegram_bot_token'
+    },
+    'ytdl_telegram_chat_id': {
+        'key': 'ytdl_telegram_chat_id',
+        'path': 'YoutubeDLMaterial.API.telegram_chat_id'
+    },
+    'ytdl_webhook_url': {
+        'key': 'ytdl_webhook_url',
+        'path': 'YoutubeDLMaterial.API.webhook_URL'
+    },
+    'ytdl_discord_webhook_url': {
+        'key': 'ytdl_discord_webhook_url',
+        'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
+    },
 
 
     // Themes
@@ -306,4 +346,6 @@ const YTDL_ARGS_WITH_VALUES = [
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
-exports.CURRENT_VERSION = 'v4.3';
+exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
+
+exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -2,6 +2,7 @@ var fs = require('fs-extra')
 var path = require('path')
 const { MongoClient } = require("mongodb");
 const { uuid } = require('uuidv4');
+const _ = require('lodash');
 
 const config_api = require('./config');
 var utils = require('./utils')
@@ -58,6 +59,13 @@ const tables = {
         name: 'tasks',
         primary_key: 'key'
     },
+    notifications: {
+        name: 'notifications',
+        primary_key: 'uid'
+    },
+    archives: {
+        name: 'archives'
+    },
     test: {
         name: 'test'
     }
@@ -148,6 +156,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
                 await database.collection(table).createIndex(text_search);
             }
         });
+        using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
         return true;
     } catch(err) {
         logger.error(err);
@@ -252,13 +261,16 @@ exports.getFileDirectoriesAndDBs = async () => {
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'audio'),
                 user_uid: user.uid,
-                type: 'audio'
+                type: 'audio',
+                archive_path: utils.getArchiveFolder('audio', user.uid)
             });
 
             // add user's video dir to check list
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'video'),
-                type: 'video'
+                user_uid: user.uid,
+                type: 'video',
+                archive_path: utils.getArchiveFolder('video', user.uid)
             });
         }
     } else {
@@ -268,13 +280,15 @@ exports.getFileDirectoriesAndDBs = async () => {
         // add audio dir to check list
         dirs_to_check.push({
             basePath: audioFolderPath,
-            type: 'audio'
+            type: 'audio',
+            archive_path: utils.getArchiveFolder('audio')
         });
 
         // add video dir to check list
         dirs_to_check.push({
             basePath: videoFolderPath,
-            type: 'video'
+            type: 'video',
+            archive_path: utils.getArchiveFolder('video')
         });
     }
 
@@ -295,7 +309,8 @@ exports.getFileDirectoriesAndDBs = async () => {
                                       : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
             user_uid: subscription_to_check.user_uid,
             type: subscription_to_check.type,
-            sub_id: subscription_to_check['id']
+            sub_id: subscription_to_check['id'],
+            archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
         });
     }
 
@@ -350,7 +365,7 @@ exports.addMetadataPropertyToDB = async (property_key) => {
             }
         }
 
-        return await exports.bulkUpdateRecords('files', 'uid', update_obj);
+        return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
     } catch(err) {
         logger.error(err);
         return false;
@@ -447,8 +462,8 @@ exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null)
     return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
 }
 
-exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid, uuid);
+exports.deleteFile = async (uid, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid);
     const type = file_obj.isAudio ? 'audio' : 'video';
     const folderPath = path.dirname(file_obj.path);
     const ext = type === 'audio' ? 'mp3' : 'mp4';
@@ -494,16 +509,22 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const archive_path = utils.getArchiveFolder(type, uuid);
+        // get id/extractor from JSON
 
-        // get ID from JSON
-
-        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let id = null;
-        if (jsonobj) id = jsonobj.id;
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedID = null;
+        let retrievedExtractor = null;
+        if (info_json) {
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
 
         // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-        await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
+        if (!blacklistMode) {
+            // workaround until a files_api is created (using archive_api would make a circular dependency)
+            await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
+            // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
+        }
     }
 
     if (jsonExists) await fs.unlink(jsonPath);
@@ -534,8 +555,32 @@ exports.getVideo = async (file_uid) => {
     return await exports.getRecord('files', {uid: file_uid});
 }
 
-exports.getFiles = async (uuid = null) => {
-    return await exports.getRecords('files', {user_uid: uuid});
+exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (favorite_filter) {
+        filter_obj['favorite'] = true;
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
+    const file_count = await exports.getRecords('files', filter_obj, true);
+
+    return {files, file_count};
 }
 
 exports.setVideoProperty = async (file_uid, assignment_obj) => {
@@ -550,7 +595,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
 exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
     // local db override
     if (using_local_db) {
-        if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
+        if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
         local_db.get(table).push(doc).write();
         return true;
     }
@@ -653,9 +698,15 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 
 // Update
 
-exports.updateRecord = async (table, filter_obj, update_obj) => {
+exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
     // local db override
     if (using_local_db) {
+        if (nested_mode) {
+            // if object is nested we need to handle it differently
+            update_obj = utils.convertFlatObjectToNestedObject(update_obj);
+            exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
+            return true;
+        }
         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
@@ -669,7 +720,14 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
+            const props_to_update = Object.keys(update_obj);
+            for (let i = 0; i < props_to_update.length; i++) {
+                const prop_to_update = props_to_update[i];
+                const prop_value = update_obj[prop_to_update];
+                record[prop_to_update] = prop_value;
+            }
+        }).write();
         return true;
     }
 
@@ -677,7 +735,19 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
     return !!(output['result']['ok']);
 }
 
-exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
+exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
+    // local db override
+    if (using_local_db) {
+        const props_to_remove = Object.keys(remove_obj);
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
+        return true;
+    }
+
+    const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
+    return !!(output['result']['ok']);
+}
+
+exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
     // local db override
     if (using_local_db) {
         local_db.get(table).each((record) => {
@@ -1091,6 +1161,14 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
                     } else if ('$ne' in filter_prop_value) {
                         filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
+                    } else if ('$lt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
+                    } else if ('$gt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
+                    } else if ('$lte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
+                    } else if ('$gte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
                     }
                 } else {
                     // handle case of nested property check
@@ -1105,3 +1183,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     });
     return return_val;
 }
+
+// should only be used for tests
+exports.setLocalDBMode = (mode) => {
+    using_local_db = mode;
+}

backend/downloader.js

@@ -1,7 +1,6 @@
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const path = require('path');
-const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
@@ -14,6 +13,8 @@ const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
+const notifications_api = require('./notifications');
+const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
@@ -26,6 +27,25 @@ if (db_api.database_initialized) {
     });
 }
 
+/*
+
+This file handles all the downloading functionality.
+
+To download a file, we go through 4 steps. Here they are with their respective index & function:
+
+0: Create the download
+ - createDownload()
+1: Get info for the download (we need this step for categories and archive functionality)
+ - collectInfo()
+2: Download the file
+ - downloadQueuedFile()
+3: Complete
+ - N/A
+
+We use checkDownloads() to move downloads through the steps and call their respective functions.
+
+*/
+
 exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
@@ -84,10 +104,10 @@ exports.resumeDownload = async (download_uid) => {
 exports.restartDownload = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     await exports.clearDownload(download_uid);
-    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
     
     should_check_downloads = true;
-    return success;
+    return new_download;
 }
 
 exports.cancelDownload = async (download_uid) => {
@@ -106,9 +126,10 @@ exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download_uid, error_message) {
-    if (!download_uid) return;
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+async function handleDownloadError(download, error_message, error_type = null) {
+    if (!download || !download['uid']) return;
+    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
+    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
 async function setupDownloads() {
@@ -154,6 +175,13 @@ async function checkDownloads() {
         if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
 
         if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            if (waiting_download['sub_id']) {
+                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
+                if (sub_missing) {
+                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    continue;
+                }
+            }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
@@ -193,6 +221,20 @@ async function collectInfo(download_uid) {
         return;
     }
 
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
+        if (exists_in_archive) {
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            logger.warn(error);
+            if (download_uid) {
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error, 'exists_in_archive');
+                return;
+            }
+        }
+    }
+
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -203,11 +245,10 @@ async function collectInfo(download_uid) {
         options.customOutput = category['custom_output'];
         options.noRelativePath = true;
         args = await exports.generateArgs(url, type, options, download['user_uid']);
-        args = utils.filterArgs(args, ['--no-simulate']);
         info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
-    download['category'] = category;
+    const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
 
     // setup info required to calculate download progress
 
@@ -230,6 +271,7 @@ async function collectInfo(download_uid) {
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
                                                                     title: playlist_title ? playlist_title : info['title'],
+                                                                    category: stripped_category,
                                                                     prefetched_info: null
                                                                 });
 }
@@ -272,14 +314,14 @@ async function downloadQueuedFile(download_uid) {
             clearInterval(download_checker);
             if (err) {
                 logger.error(err.stderr);
-                await handleDownloadError(download_uid, err.stderr);
+                await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_output');
                     logger.warn(error_message);
                     resolve(false);
                     return;
@@ -288,7 +330,10 @@ async function downloadQueuedFile(download_uid) {
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
-                        output_json = JSON.parse(output[i]);
+                        // we have to do this because sometimes there will be leading characters before the actual json
+                        const start_idx = output[i].indexOf('{"');
+                        const clean_output = output[i].slice(start_idx, output[i].length);
+                        output_json = JSON.parse(clean_output);
                     } catch(e) {
                         output_json = null;
                     }
@@ -305,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -341,15 +386,12 @@ async function downloadQueuedFile(download_uid) {
                     // registers file in DB
                     const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
-                    file_objs.push(file_obj);
-                }
+                    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
 
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
+                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
+
+                    file_objs.push(file_obj);
                 }
 
                 let container = null;
@@ -363,7 +405,7 @@ async function downloadQueuedFile(download_uid) {
                 } else {
                     const error_message = 'Downloaded file failed to result in metadata object.';
                     logger.error(error_message);
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_metadata');
                 }
 
                 const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -379,6 +421,10 @@ async function downloadQueuedFile(download_uid) {
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
 
+    if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
+        logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
+    }
+
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
     const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
@@ -426,7 +472,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
         } else if (heightParam && heightParam !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height${maxHeight ? '<' : ''}=${heightParam}]`];
+            const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
+            qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -463,28 +510,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -527,7 +552,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
-        const new_args = [...args];
+        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...temp_args];
 
         const archiveArgIndex = new_args.indexOf('--download-archive');
         if (archiveArgIndex !== -1) {
@@ -559,7 +585,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                     const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
                     logger.error(error);
                     if (download_uid) {
-                        await handleDownloadError(download_uid, error);
+                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download, error, 'parse_failed');
                     }
                     resolve(null);
                 }
@@ -568,7 +595,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                 if (err.stderr) error_message += `\n\n${err.stderr}`;
                 logger.error(error_message);
                 if (download_uid) {
-                    await handleDownloadError(download_uid, error_message);
+                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
                 }
                 resolve(null);
             }
@@ -634,13 +662,3 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join('appdata', 'archives');
-    }
-}

backend/entrypoint.sh

@@ -10,7 +10,7 @@ fi
 
 # chown current working directory to current user
 if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
   exec gosu "$UID:$GID" "$0" "$@"
 fi
 

backend/notifications.js

@@ -0,0 +1,187 @@
+const db_api = require('./db');
+const config_api = require('./config');
+const logger = require('./logger');
+const utils = require('./utils');
+const consts = require('./consts');
+
+const { uuid } = require('uuidv4');
+
+const fetch = require('node-fetch');
+const { gotify } = require("gotify");
+const TelegramBot = require('node-telegram-bot-api');
+const REST = require('@discordjs/rest').REST;
+const API = require('@discordjs/core').API;
+const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
+
+const NOTIFICATION_TYPE_TO_TITLE = {
+    task_finished: 'Task finished',
+    download_complete: 'Download complete',
+    download_error: 'Download error'
+}
+
+const NOTIFICATION_TYPE_TO_BODY = {
+    task_finished: (notification) => notification['data']['task_title'],
+    download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
+    download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
+}
+
+const NOTIFICATION_TYPE_TO_URL = {
+    task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
+    download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
+    download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
+}
+
+const NOTIFICATION_TYPE_TO_THUMBNAIL = {
+    task_finished: () => null,
+    download_complete: (notification) => notification['data']['file_thumbnail'],
+    download_error: () => null
+}
+
+exports.sendNotification = async (notification) => {
+    // info necessary if we are using 3rd party APIs
+    const type = notification['type'];
+
+    const data = {
+        title: NOTIFICATION_TYPE_TO_TITLE[type],
+        body: NOTIFICATION_TYPE_TO_BODY[type](notification),
+        type: type,
+        url: NOTIFICATION_TYPE_TO_URL[type](notification),
+        thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
+    }
+
+    if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
+        sendNtfyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
+        sendGotifyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
+        sendTelegramNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_webhook_url')) {
+        sendGenericNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
+        sendDiscordNotification(data);
+    }
+
+    await db_api.insertRecordIntoTable('notifications', notification);
+    return notification;
+}
+
+exports.sendTaskNotification = async (task_obj, confirmed) => {
+    if (!notificationEnabled('task_finished')) return;
+    // workaround for tasks which are user_uid agnostic
+    const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
+    await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
+    const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
+    const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadNotification = async (file, user_uid) => {
+    if (!notificationEnabled('download_complete')) return;
+    const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
+    const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
+    if (!notificationEnabled('download_error')) return;
+    const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
+    const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.createNotification = (type, actions, data, user_uid) => {
+    const notification = {
+        type: type,
+        actions: actions,
+        data: data,
+        user_uid: user_uid,
+        uid: uuid(),
+        read: false,
+        timestamp: Date.now()/1000
+    }
+    return notification;
+}
+
+function notificationEnabled(type) {
+    return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
+}
+
+function sendNtfyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to ntfy');
+    fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
+        method: 'POST',
+        body: body,
+        headers: {
+            'Title': title,
+            'Tags': type,
+            'Click': url,
+            'Attach': thumbnail
+        }
+    });
+}
+
+async function sendGotifyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to gotify');
+    await gotify({
+        server: config_api.getConfigItem('ytdl_gotify_server_url'),
+        app: config_api.getConfigItem('ytdl_gotify_app_token'),
+        title: title,
+        message: body,
+        tag: type,
+        priority: 5, // Keeping default from docs, may want to change this,
+        extras: {
+            "client::notification": {
+                click: { url: url },
+                bigImageUrl: thumbnail
+            }
+        }
+      });
+}
+
+async function sendTelegramNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to Telegram');
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    const bot = new TelegramBot(bot_token);
+    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+}
+
+async function sendDiscordNotification({body, title, type, url, thumbnail}) {
+    const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
+    const url_split = discord_webhook_url.split('webhooks/');
+    const [webhook_id, webhook_token] = url_split[1].split('/');
+    const rest = new REST({ version: '10' });
+    const api = new API(rest);
+    const embed = new EmbedBuilder()
+        .setTitle(title)
+        .setColor(0x00FFFF)
+        .setURL(url)
+        .setDescription(`ID: ${type}`);
+    if (thumbnail) embed.setThumbnail(thumbnail);
+    if (type === 'download_error') embed.setColor(0xFC2003);
+
+    const result = await api.webhooks.execute(webhook_id, webhook_token, {
+        content: body,
+        username: 'YoutubeDL-Material',
+        avatar_url: consts.ICON_URL,
+        embeds: [embed],
+    });
+    return result;
+}
+
+function sendGenericNotification(data) {
+    const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
+    logger.verbose(`Sending generic notification to ${webhook_url}`);
+    fetch(webhook_url, {
+        method: 'POST',
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify(data),
+    });
+}

backend/package.json

@@ -19,21 +19,25 @@
   },
   "homepage": "",
   "dependencies": {
+    "@discordjs/builders": "^1.6.1",
+    "@discordjs/core": "^0.5.2",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
-    "async-mutex": "^0.3.1",
+    "async-mutex": "^0.4.0",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
     "express-session": "^1.17.3",
+    "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "gotify": "^1.1.0",
     "jsonwebtoken": "^8.5.1",
+    "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
-    "merge-files": "^0.1.2",
     "mocha": "^9.2.2",
     "moment": "^2.29.4",
     "mongodb": "^3.6.9",
@@ -41,9 +45,10 @@
     "node-fetch": "^2.6.7",
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
-    "passport": "^0.6.0",
+    "node-telegram-bot-api": "^0.61.0",
+    "passport": "^0.4.1",
     "passport-http": "^0.3.0",
-    "passport-jwt": "^4.0.0",
+    "passport-jwt": "^4.0.1",
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
@@ -52,7 +57,7 @@
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.0.6",
+    "uuidv4": "^6.2.13",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"

backend/subscriptions.js

@@ -3,6 +3,7 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
+const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
 
@@ -91,7 +92,10 @@ async function getSubscriptionInfo(sub) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
+                            let sub_name = sub.name;
+                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
+                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
                         }
                     }
 
@@ -138,28 +142,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
 
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     if (deleteMode && (await fs.pathExists(appendedBasePath))) {
-        if (sub.archive && (await fs.pathExists(sub.archive))) {
-            const archive_file_path = path.join(sub.archive, 'archive.txt');
-            // deletes archive if it exists
-            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
-            if (await fs.pathExists(archive_file_path)) {
-                await fs.unlink(archive_file_path);
-            }
-            await fs.rmdir(sub.archive);
-        }
         await fs.remove(appendedBasePath);
     }
+
+    await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
 async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    if (typeof sub === 'string') {
+        // TODO: fix bad workaround where sub is a sub_id
+        sub = await db_api.getRecord('subscriptions', {sub_id: sub});
+    }
     // TODO: combine this with deletefile
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
+    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -178,7 +179,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = fs.readJSONSync(jsonPath)['id'];
+        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = info_json['id'];
+        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -196,11 +199,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (useArchive && retrievedID) {
-                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
-
-                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
+            const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useArchive && !deleteForever) {
+                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
         }
@@ -231,13 +232,20 @@ async function getVideosForSub(sub, user_uid = null) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
@@ -331,8 +339,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -358,6 +364,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
+    // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
+        const archive_path = path.join(appendedBasePath, 'archive.txt');
+        await fs.writeFile(archive_path, archive_text);
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
@@ -368,21 +384,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
-    let archive_dir = null;
-    let archive_path = null;
-
-    if (useArchive && !redownload) {
-        if (sub.archive) {
-            archive_dir = sub.archive;
-            if (sub.type && sub.type === 'audio') {
-                archive_path = path.join(archive_dir, 'merged_audio.txt');
-            } else {
-                archive_path = path.join(archive_dir, 'merged_video.txt');
-            }            
-        }
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -425,7 +426,14 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+                continue;
             }
+            const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useYoutubeDLArchive) {
+                const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+                if (exists_in_archive) continue;
+            }
+
             files_to_download.push(output_json);
         }
     }
@@ -444,7 +452,12 @@ async function getAllSubscriptions() {
 }
 
 async function getSubscription(subID) {
-    return await db_api.getRecord('subscriptions', {id: subID});
+    // stringify and parse because we may override the 'downloading' property
+    const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
+    // now with the download_queue, we may need to override 'downloading'
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
+    return sub;
 }
 
 async function getSubscriptionByName(subName, user_uid = null) {

backend/tasks.js

@@ -1,5 +1,7 @@
 const db_api = require('./db');
+const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 const fs = require('fs-extra');
 const logger = require('./logger');
@@ -33,6 +35,28 @@ const TASKS = {
         confirm: youtubedl_api.updateYoutubeDL,
         title: 'Update youtube-dl',
         job: null
+    },
+    delete_old_files: {
+        run: checkForAutoDeleteFiles,
+        confirm: autoDeleteFiles,
+        title: 'Delete old files',
+        job: null
+    },
+    import_legacy_archives: {
+        run: archive_api.importArchives,
+        title: 'Import legacy archives',
+        job: null
+    }
+}
+
+const defaultOptions = {
+    all: {
+        auto_confirm: false
+    },
+    delete_old_files: {
+        blacklist_files: false,
+        blacklist_subscription_files: false,
+        threshold_days: ''
     }
 }
 
@@ -45,7 +69,7 @@ function scheduleJob(task_key, schedule) {
         const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
         const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
         const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
         return null;
@@ -57,7 +81,7 @@ function scheduleJob(task_key, schedule) {
             logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
             return;
         }
-        
+
         // remove schedule if it's a one-time task
         if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
         // we're just "running" the task, any confirmation should be user-initiated
@@ -77,9 +101,10 @@ exports.setupTasks = async () => {
     const tasks_keys = Object.keys(TASKS);
     for (let i = 0; i < tasks_keys.length; i++) {
         const task_key = tasks_keys[i];
+        const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
         const task_in_db = await db_api.getRecord('tasks', {key: task_key});
         if (!task_in_db) {
-            // insert task metadata into table if missing
+            // insert task metadata into table if missing, eventually move title to UI
             await db_api.insertRecordIntoTable('tasks', {
                 key: task_key,
                 title: TASKS[task_key]['title'],
@@ -90,9 +115,19 @@ exports.setupTasks = async () => {
                 data: null,
                 error: null,
                 schedule: null,
-                options: {}
+                options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
             });
         } else {
+            // verify all options exist in task
+            for (const key of Object.keys(mergedDefaultOptions)) {
+                const option_key = `options.${key}`;
+                // Remove any potential mangled option keys (#861)
+                await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
+                if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
+                    await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
+                }
+            }
+
             // reset task if necessary
             await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
 
@@ -123,15 +158,23 @@ exports.executeTask = async (task_key) => {
 
 exports.executeRun = async (task_key) => {
     logger.verbose(`Running task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     // don't set running to true when backup up DB as it will be stick "running" if restored
     if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
     const data = await TASKS[task_key].run();
     await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
     logger.verbose(`Finished running task ${task_key}`);
+    const task_obj = await db_api.getRecord('tasks', {key: task_key});
+    await notifications_api.sendTaskNotification(task_obj, false);
+
+    if (task_obj['options'] && task_obj['options']['auto_confirm']) {
+        exports.executeConfirm(task_key);
+    }
 }
 
 exports.executeConfirm = async (task_key) => {
     logger.verbose(`Confirming task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     if (!TASKS[task_key]['confirm']) {
         return null;
     }
@@ -141,6 +184,7 @@ exports.executeConfirm = async (task_key) => {
     await TASKS[task_key].confirm(data);
     await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
     logger.verbose(`Finished confirming task ${task_key}`);
+    await notifications_api.sendTaskNotification(task_obj, false);
 }
 
 exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -193,4 +237,31 @@ async function removeDuplicates(data) {
     }
 }
 
+// auto delete files
+
+async function checkForAutoDeleteFiles() {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
+        const error_message = 'Failed to do delete check because no limit was set!';
+        logger.error(error_message);
+        await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
+        return null;
+    }
+    const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
+    const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
+    const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
+    return {files_to_remove: files_to_remove};
+}
+
+async function autoDeleteFiles(data) {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (data['files_to_remove']) {
+        logger.info(`Removing ${data['files_to_remove'].length} old files!`);
+        for (let i = 0; i < data['files_to_remove'].length; i++) {
+            const file_to_remove = data['files_to_remove'][i];
+            await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
+        }
+    }
+}
+
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -1,9 +1,9 @@
+/* eslint-disable no-undef */
 const assert = require('assert');
 const low = require('lowdb')
 const winston = require('winston');
 const path = require('path');
 
-process.chdir('./backend')
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -38,6 +38,8 @@ var auth_api = require('../authentication/auth');
 var db_api = require('../db');
 const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
+const archive_api = require('../archive');
+const categories_api = require('../categories');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const NodeID3 = require('node-id3');
@@ -66,12 +68,12 @@ const sample_video_json = {
 
 describe('Database', async function() {
     describe('Import', async function() {
-        it('Migrate', async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords();
-            const success = await db_api.importJSONToDB(db.value(), users_db.value());
-            assert(success);
-        });
+        // it('Migrate', async function() {
+        //     await db_api.connectToDB();
+        //     await db_api.removeAllRecords();
+        //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
+        //     assert(success);
+        // });
 
         it('Transfer to remote', async function() {
             await db_api.removeAllRecords('test');
@@ -104,157 +106,208 @@ describe('Database', async function() {
         });
     });
 
-    describe('Export', function() {
-
-    });
-
-
     describe('Basic functions', async function() {
-        beforeEach(async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords('test');
-        });
-        it('Add and read record', async function() {
-            this.timeout(120000);
-            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
-            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
-            assert(added_record['test_add'] === 'test');
-            await db_api.removeRecord('test', {test_add: 'test'});
-        });
+        
+        // test both local_db and remote_db
+        const local_db_modes = [false, true];
 
-        it('Find duplicates by key', async function() {
-            const test_duplicates = [
-                {
-                    test: 'testing',
-                    key: '1'
-                },
-                {
-                    test: 'testing',
-                    key: '2'
-                },
-                {
-                    test: 'testing_missing',
-                    key: '3'
-                },
-                {
-                    test: 'testing',
-                    key: '4'
-                }
-            ];
-            await db_api.insertRecordsIntoTable('test', test_duplicates);
-            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-            console.log(duplicates);
-        });
-
-        it('Update record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
-            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
-            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
-            assert(updated_record['added_field']);
-            await db_api.removeRecord('test', {test_update: 'test'});
-        });
-
-        it('Remove record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
-            assert(delete_succeeded);
-            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
-            assert(!deleted_record);
-        });
-
-        it('Push to record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
-            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 1);
-        });
-
-        it('Pull from record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
-            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 0);
-        });
-
-        it('Bulk add', async function() {
-            this.timeout(120000);
-            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
-            const test_records = [];
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                test_records.push({
-                    uid: uuid()
+        for (const local_db_mode of local_db_modes) {
+            let use_local_db = local_db_mode;
+            describe(`Use local DB - ${use_local_db}`, async function() {
+                beforeEach(async function() {
+                    if (!use_local_db) {
+                        this.timeout(120000);
+                        await db_api.connectToDB(0);
+                    }
+                    await db_api.removeAllRecords('test');
                 });
-            }
-            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-
-            const received_records = await db_api.getRecords('test');
-            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
-        });
-
-        it('Bulk update', async function() {
-            // bulk add records
-            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
-            const test_records = [];
-            const update_obj = {};
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const test_uid =  uuid();
-                test_records.push({
-                    uid: test_uid
+                it('Add and read record', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+                    assert(added_record['test_add'] === 'test');
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Add and read record - Nested property', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
+                    const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
+                    assert(added_record['test_add'] === 'test');
+                    assert(!not_added_record);
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Replace filter', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
+                    assert(count === 1);
+                    await db_api.removeRecord('test', {test_replace_filter: 'test'});
+                });
+                it('Find duplicates by key', async function() {
+                    const test_duplicates = [
+                        {
+                            test: 'testing',
+                            key: '1'
+                        },
+                        {
+                            test: 'testing',
+                            key: '2'
+                        },
+                        {
+                            test: 'testing_missing',
+                            key: '3'
+                        },
+                        {
+                            test: 'testing',
+                            key: '4'
+                        }
+                    ];
+                    await db_api.insertRecordsIntoTable('test', test_duplicates);
+                    const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+                    console.log(duplicates);
                 });
-                update_obj[test_uid] = {added_field: true};
-            }
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            assert(success);
 
-            // makes sure they are added
-            const received_records = await db_api.getRecords('test');
-            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                it('Update record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+                    await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+                    const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+                    assert(updated_record['added_field']);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
-            assert(success);
+                it('Update records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
+                    await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
+                    const updated_records = await db_api.getRecords('test', {added_field: true});
+                    assert(updated_records.length === 2);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            const received_updated_records = await db_api.getRecords('test');
-            for (let i = 0; i < received_updated_records.length; i++) {
-                success &= received_updated_records[i]['added_field'];
-            }
-            assert(success);
-        });
+                it('Remove property from record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
+                    await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
+                    const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
+                    assert(updated_record['test_keep']);
+                    assert(!updated_record['test_remove']);
+                    await db_api.removeRecord('test', {test_keep: 'test'});
+                });
 
-        it('Stats', async function() {
-            const stats = await db_api.getDBStats();
-            assert(stats);
-        });
+                it('Remove record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+                    assert(!deleted_record);
+                });
 
-        it('Query speed', async function() {
-            this.timeout(120000); 
-            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
-            const test_records = [];
-            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const uid = uuid();
-                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
-            }
-            const insert_start = Date.now();
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            const insert_end = Date.now();
+                it('Remove records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
+                    assert(count === 0);
+                });
 
-            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+                it('Push to record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+                    await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 1);
+                });
 
-            const query_start = Date.now();
-            const random_record = await db_api.getRecord('test', {uid: random_uid});
-            const query_end = Date.now();
+                it('Pull from record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+                    await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 0);
+                });
 
-            console.log(random_record)
+                it('Bulk add', async function() {
+                    this.timeout(120000);
+                    const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+                    const test_records = [];
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        test_records.push({
+                            uid: uuid()
+                        });
+                    }
+                    const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
 
-            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+                    const received_records = await db_api.getRecords('test');
+                    assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                });
 
-            success = !!random_record;
+                it('Bulk update', async function() {
+                    // bulk add records
+                    const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+                    const test_records = [];
+                    const update_obj = {};
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const test_uid =  uuid();
+                        test_records.push({
+                            uid: test_uid
+                        });
+                        update_obj[test_uid] = {added_field: true};
+                    }
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    assert(success);
 
-            assert(success);
-        });
+                    // makes sure they are added
+                    const received_records = await db_api.getRecords('test');
+                    assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+                    success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
+                    assert(success);
+
+                    const received_updated_records = await db_api.getRecords('test');
+                    for (let i = 0; i < received_updated_records.length; i++) {
+                        success &= received_updated_records[i]['added_field'];
+                    }
+                    assert(success);
+                });
+
+                it('Stats', async function() {
+                    const stats = await db_api.getDBStats();
+                    assert(stats);
+                });
+
+                it('Query speed', async function() {
+                    this.timeout(120000); 
+                    const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+                    const test_records = [];
+                    let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const uid = uuid();
+                        if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                        test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                    }
+                    const insert_start = Date.now();
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    const insert_end = Date.now();
+
+                    console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+                    const query_start = Date.now();
+                    const random_record = await db_api.getRecord('test', {uid: random_uid});
+                    const query_end = Date.now();
+
+                    console.log(random_record)
+
+                    console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+                    success = !!random_record;
+
+                    assert(success);
+                });
+            });
+        }
     });
 
     describe('Local DB Filters', async function() {
@@ -289,8 +342,6 @@ describe('Multi User', async function() {
     const playlist_to_test = 'ysabVZz4x';
     beforeEach(async function() {
         await db_api.connectToDB();
-        auth_api.initialize(db_api, logger);
-        subscriptions_api.initialize(db_api, logger);
         user = await auth_api.login('admin', 'pass');
     });
     describe('Authentication', function() {
@@ -299,8 +350,10 @@ describe('Multi User', async function() {
         });
     });
     describe('Video player - normal', async function() {
-        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
-        await db_api.insertRecordIntoTable('files', sample_video_json);
+        beforeEach(async function() {
+            await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+            await db_api.insertRecordIntoTable('files', sample_video_json);
+        });
         const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
             const video_obj = await db_api.getVideo(video_to_test);
@@ -457,18 +510,23 @@ describe('Downloader', function() {
         const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
         const updated_args1 = utils.injectArgs(original_args1, new_args1);
         const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
-        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
+        assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
 
         const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
         const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
         const updated_args2 = utils.injectArgs(original_args2, new_args2);
-        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
-        console.log(updated_args2);
-        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
+
+        const original_args3 = ['-o', '%(title)s.%(ext)s'];
+        const new_args3 =  ['--min-filesize','1'];
+        const updated_args3 = utils.injectArgs(original_args3, new_args3);
+        const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
+        assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
     });
     describe('Twitch', async function () {
         const twitch_api = require('../twitch');
-        const example_vod = '1493770675';
+        const example_vod = '1710641401';
         it('Download VOD', async function() {
             const sample_path = path.join('test', 'sample.twitch_chat.json');
             if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
@@ -550,7 +608,7 @@ describe('Tasks', function() {
         fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file, true);
+        assert(!!imported_file === true);
         
         // post-test cleanup
         if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -593,30 +651,72 @@ describe('Tasks', function() {
 });
 
 describe('Archive', async function() {
-    const archive_path = path.join('test', 'archives');
-    fs.ensureDirSync(archive_path);
-    const archive_file_path = path.join(archive_path, 'archive_video.txt');
-    const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
     beforeEach(async function() {
-        if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
-        fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
-
-        if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
-        fs.writeFileSync(blacklist_file_path, '');
-    });
-    
-    it('Delete from archive', async function() {
-        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
-        const new_archive = fs.readFileSync(archive_file_path);
-        assert(!new_archive.includes('testing2'));
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
     });
 
-    it('Delete from archive - blacklist', async function() {
-        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
-        const new_archive = fs.readFileSync(archive_file_path);
-        const new_blacklist = fs.readFileSync(blacklist_file_path);
-        assert(!new_archive.includes('testing2'));
-        assert(new_blacklist.includes('testing2'));
+    afterEach(async function() {
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    it('Import archive', async function() {
+        const archive_text = `
+            testextractor1 testing1
+            testextractor1 testing2
+            testextractor2 testing1
+            testextractor1 testing3
+
+        `;
+        const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
+        assert(count === 4)
+        const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        console.log(archive_items);
+        assert(archive_items.length === 4);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
+
+        const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        assert(success);
+    });
+
+    it('Get archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+
+        assert(archive_item1 && archive_item2);
+    });
+
+    it('Archive duplicates', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
+
+        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
+        assert(count === 3);
+    });
+
+    it('Remove from archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
+
+        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
+        assert(success);
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        assert(!!archive_item1);
+        
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+        assert(!archive_item2);
+
+        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
+        assert(!!archive_item3);
     });
 });
 
@@ -626,4 +726,129 @@ describe('Utils', async function() {
         const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
         assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
     });
+
+    it('Convert flat object to nested object', async function() {
+        // No modfication
+        const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
+        const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
+        assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
+        assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
+
+        // Standard setup
+        const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
+        const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
+        assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
+        assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
+
+        // Nested branches
+        const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
+        const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
+    });
+});
+
+describe('Categories', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        const new_category = {
+            name: 'test_category',
+            uid: uuid(),
+            rules: [],
+            custom_output: ''
+        };
+    
+        await db_api.insertRecordIntoTable('categories', new_category);
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('categories', {name: 'test_category'});
+    });
+
+    it('Categorize - includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        console.log(category);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - AND', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'and',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - OR', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'or',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category);
+    });
 });

backend/twitch.js

@@ -4,19 +4,28 @@ const logger = require('./logger');
 const moment = require('moment');
 const fs = require('fs-extra')
 const path = require('path');
+const { promisify } = require('util');
+const child_process = require('child_process');
 
-async function getCommentsForVOD(clientID, clientSecret, vodId) {
-    const { promisify } = require('util');
-    const child_process = require('child_process');
+async function getCommentsForVOD(vodId) {
     const exec = promisify(child_process.exec);
     
     // Reject invalid params to prevent command injection attack
-    if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
-        logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
+    if (!vodId.match(/^[0-9a-z]+$/)) {
+        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
         return null;
     }
 
-    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
+    const is_windows = process.platform === 'win32';
+    const cliExt = is_windows ? '.exe' : ''
+    const cliPath = `TwitchDownloaderCLI${cliExt}`
+
+    if (!fs.existsSync(cliPath)) {
+        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
+        return null;
+    }
+
+    const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
 
     if (result['stderr']) {
         logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -73,9 +82,7 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
 async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
     const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
     const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
-    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
-    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
+    const chat = await getCommentsForVOD(vodId);
 
     // save file if needed params are included
     let file_path = null;

backend/utils.js

@@ -4,6 +4,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const archiver = require('archiver');
 const fetch = require('node-fetch');
 const ProgressBar = require('progress');
+const winston = require('winston');
 
 const config_api = require('./config');
 const logger = require('./logger');
@@ -12,7 +13,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-function getTrueFileName(unfixed_path, type) {
+exports.getTrueFileName = (unfixed_path, type) => {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -27,13 +28,13 @@ function getTrueFileName(unfixed_path, type) {
     return fixed_path;
 }
 
-async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
+exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
     // return empty array if the path doesn't exist
     if (!(await fs.pathExists(basePath))) return [];
 
     let files = [];
     const ext = type === 'audio' ? 'mp3' : 'mp4';
-    var located_files = await recFindByExt(basePath, ext);
+    var located_files = await exports.recFindByExt(basePath, ext);
     for (let i = 0; i < located_files.length; i++) {
         let file = located_files[i];
         var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -41,33 +42,33 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
         var stats = await fs.stat(file);
 
         var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await getJSONByType(type, id, basePath);
+        var jsonobj = await exports.getJSONByType(type, id, basePath);
         if (!jsonobj) continue;
         if (full_metadata) {
             jsonobj['id'] = id;
             files.push(jsonobj);
             continue;
         }
-        var upload_date = formatDateString(jsonobj.upload_date);
+        var upload_date = exports.formatDateString(jsonobj.upload_date);
 
         var isaudio = type === 'audio';
-        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+        var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
                                 stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
 }
 
-async function createContainerZipFile(file_name, container_file_objs) {
+exports.createContainerZipFile = async (file_name, container_file_objs) => {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
 }
 
-async function createZipFile(zip_file_path, file_paths) {
+exports.createZipFile = async (zip_file_path, file_paths) => {
     let output = fs.createWriteStream(zip_file_path);
 
     var archive = archiver('zip', {
@@ -91,11 +92,11 @@ async function createZipFile(zip_file_path, file_paths) {
     await archive.finalize();
 
     // wait a tiny bit for the zip to reload in fs
-    await wait(100);
+    await exports.wait(100);
     return zip_file_path;
 }
 
-function getJSONMp4(name, customPath, openReadPerms = false) {
+exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -110,7 +111,7 @@ function getJSONMp4(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSONMp3(name, customPath, openReadPerms = false) {
+exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
     var obj = null;
     if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -127,11 +128,11 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSON(file_path, type) {
+exports.getJSON = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
     let obj = null;
-    var jsonPath = removeFileExtension(file_path) + '.info.json';
-    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
+    var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
     if (fs.existsSync(jsonPath))
     {
         obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -142,12 +143,12 @@ function getJSON(file_path, type) {
     return obj;
 }
 
-function getJSONByType(type, name, customPath, openReadPerms = false) {
-    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
+    return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(file_path) {
-    const file_path_no_extension = removeFileExtension(file_path);
+exports.getDownloadedThumbnail = (file_path) => {
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
     let webpPath = file_path_no_extension + '.webp';
@@ -163,7 +164,7 @@ function getDownloadedThumbnail(file_path) {
         return null;
 }
 
-function getExpectedFileSize(input_info_jsons) {
+exports.getExpectedFileSize = (input_info_jsons) => {
     // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
     const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
 
@@ -186,12 +187,12 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(file_path, type) {
+exports.fixVideoMetadataPerms = (file_path, type) => {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     const files_to_fix = [
         // JSONs
@@ -208,10 +209,10 @@ function fixVideoMetadataPerms(file_path, type) {
     }
 }
 
-function deleteJSONFile(file_path, type) {
+exports.deleteJSONFile = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
@@ -220,58 +221,7 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-// archive helper functions
-
-async function removeIDFromArchive(archive_path, type, id) {
-    const archive_file = path.join(archive_path, `archive_${type}.txt`);
-    const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
-    if (!data) {
-        logger.error('Archive could not be found.');
-        return;
-    }
-
-    let dataArray = data.split('\n'); // convert file data in an array
-    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-    let lastIndex = -1; // let say, we have not found the keyword
-
-    for (let index=0; index<dataArray.length; index++) {
-        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-            lastIndex = index; // found a line includes a id keyword
-            break;
-        }
-    }
-
-    if (lastIndex === -1) return null;
-
-    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-
-    // UPDATE FILE WITH NEW DATA
-    const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_file, updatedData);
-    if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
-}
-
-async function writeToBlacklist(archive_folder, type, line) {
-    let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}
-
-async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
-    const archive_file = path.join(archive_path, `archive_${type}.txt`);
-    if (await fs.pathExists(archive_path)) {
-        const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
-        if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
-    } else {
-        logger.info(`Could not find archive file for file ${uid}. Creating...`);
-        await fs.close(await fs.open(archive_file, 'w'));
-    }
-}
-
-function durationStringToNumber(dur_str) {
+exports.durationStringToNumber = (dur_str) => {
     if (typeof dur_str === 'number') return dur_str;
     let num_sum = 0;
     const dur_str_parts = dur_str.split(':');
@@ -281,23 +231,22 @@ function durationStringToNumber(dur_str) {
     return num_sum;
 }
 
-function getMatchingCategoryFiles(category, files) {
+exports.getMatchingCategoryFiles = (category, files) => {
     return files && files.filter(file => file.category && file.category.uid === category.uid);
 }
 
-function addUIDsToCategory(category, files) {
-    const files_that_match = getMatchingCategoryFiles(category, files);
+exports.addUIDsToCategory = (category, files) => {
+    const files_that_match = exports.getMatchingCategoryFiles(category, files);
     category['uids'] = files_that_match.map(file => file.uid);
     return files_that_match;
 }
 
-function getCurrentDownloader() {
+exports.getCurrentDownloader = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     return details_json['downloader'];
 }
 
-async function recFindByExt(base, ext, files, result, recursive = true)
-{
+exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
     files = files || (await fs.readdir(base))
     result = result || []
 
@@ -306,7 +255,7 @@ async function recFindByExt(base, ext, files, result, recursive = true)
         if ( (await fs.stat(newbase)).isDirectory() )
         {
             if (!recursive) continue;
-            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+            result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
         {
@@ -319,17 +268,17 @@ async function recFindByExt(base, ext, files, result, recursive = true)
     return result
 }
 
-function removeFileExtension(filename) {
+exports.removeFileExtension = (filename) => {
     const filename_parts = filename.split('.');
     filename_parts.splice(filename_parts.length - 1);
     return filename_parts.join('.');
 }
 
-function formatDateString(date_string) {
+exports.formatDateString = (date_string) => {
     return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
 }
 
-function createEdgeNGrams(str) {
+exports.createEdgeNGrams = (str) => {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
@@ -351,7 +300,7 @@ function createEdgeNGrams(str) {
 
 // ffmpeg helper functions
 
-async function cropFile(file_path, start, end, ext) {
+exports.cropFile = async (file_path, start, end, ext) => {
     return new Promise(resolve => {
         const temp_file_path = `${file_path}.cropped${ext}`;
         let base_ffmpeg_call = ffmpeg(file_path);
@@ -380,13 +329,13 @@ async function cropFile(file_path, start, end, ext) {
  * setTimeout, but its a promise.
  * @param {number} ms
  */
- async function wait(ms) {
+exports.wait = async (ms) => {
     await new Promise(resolve => {
         setTimeout(resolve, ms);
     });
 }
 
-async function checkExistsWithTimeout(filePath, timeout) {
+exports.checkExistsWithTimeout = async (filePath, timeout) => {
     return new Promise(function (resolve, reject) {
 
         var timer = setTimeout(function () {
@@ -415,7 +364,7 @@ async function checkExistsWithTimeout(filePath, timeout) {
 }
 
 // helper function to download file using fetch
-async function fetchFile(url, path, file_label) {
+exports.fetchFile = async (url, path, file_label) => {
     var len = null;
     const res = await fetch(url);
 
@@ -442,7 +391,7 @@ async function fetchFile(url, path, file_label) {
     });
 }
 
-async function restartServer(is_update = false) {
+exports.restartServer = async (is_update = false) => {
     logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
 
     // the following line restarts the server through pm2
@@ -455,7 +404,7 @@ async function restartServer(is_update = false) {
 //  - if already exists and doesn't have value, ignore
 //  - if it doesn't exist and has value, add both arg and value
 //  - if it doesn't exist and doesn't have value, add arg
-function injectArgs(original_args, new_args) {
+exports.injectArgs = (original_args, new_args) => {
     const updated_args = original_args.slice();
     try {
         for (let i = 0; i < new_args.length; i++) {
@@ -465,10 +414,11 @@ function injectArgs(original_args, new_args) {
             if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
                 if (original_args.includes(new_arg)) {
                     const original_index = original_args.indexOf(new_arg);
-                    original_args.splice(original_index, 2);
+                    updated_args.splice(original_index, 2);
                 }
 
                 updated_args.push(new_arg, new_args[i + 1]);
+                i++; // we need to skip the arg value on the next loop
             } else {
                 if (!original_args.includes(new_arg)) {
                     updated_args.push(new_arg);
@@ -483,11 +433,11 @@ function injectArgs(original_args, new_args) {
     return updated_args;
 }
 
-function filterArgs(args, args_to_remove) {
+exports.filterArgs = (args, args_to_remove) => {
     return args.filter(x => !args_to_remove.includes(x));
 }
 
-const searchObjectByString = function(o, s) {
+exports.searchObjectByString = (o, s) => {
     s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
     s = s.replace(/^\./, '');           // strip a leading dot
     var a = s.split('.');
@@ -502,7 +452,7 @@ const searchObjectByString = function(o, s) {
     return o;
 }
 
-function stripPropertiesFromObject(obj, properties, whitelist = false) {
+exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
     if (!whitelist) {
         const new_obj = JSON.parse(JSON.stringify(obj));
         for (let field of properties) {
@@ -518,7 +468,7 @@ function stripPropertiesFromObject(obj, properties, whitelist = false) {
     return new_obj;
 }
 
-function getArchiveFolder(type, user_uid = null, sub = null) {
+exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
     const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
     const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
@@ -537,6 +487,38 @@ function getArchiveFolder(type, user_uid = null, sub = null) {
     }
 }
 
+exports.getBaseURL = () => {
+    return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
+}
+
+exports.updateLoggerLevel = (new_logger_level) => {
+    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    if (!possible_levels.includes(new_logger_level)) {
+        logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
+        new_logger_level = 'info';
+    }
+    logger.level = new_logger_level;
+    winston.loggers.get('console').level = new_logger_level;
+    logger.transports[2].level = new_logger_level;
+}
+
+exports.convertFlatObjectToNestedObject = (obj) => {
+    const result = {};
+    for (const key in obj) {
+      const nestedKeys = key.split('.');
+      let currentObj = result;
+      for (let i = 0; i < nestedKeys.length; i++) {
+        if (i === nestedKeys.length - 1) {
+          currentObj[nestedKeys[i]] = obj[key];
+        } else {
+          currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
+          currentObj = currentObj[nestedKeys[i]];
+        }
+      }
+    }
+    return result;
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -554,38 +536,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.view_count = view_count;
     this.height = height;
     this.abr = abr;
-}
+    this.favorite = false;
+}   
+exports.File = File;
 
-module.exports = {
-    getJSONMp3: getJSONMp3,
-    getJSONMp4: getJSONMp4,
-    getJSON: getJSON,
-    getTrueFileName: getTrueFileName,
-    getDownloadedThumbnail: getDownloadedThumbnail,
-    getExpectedFileSize: getExpectedFileSize,
-    fixVideoMetadataPerms: fixVideoMetadataPerms,
-    deleteJSONFile: deleteJSONFile,
-    removeIDFromArchive: removeIDFromArchive,
-    writeToBlacklist: writeToBlacklist,
-    deleteFileFromArchive: deleteFileFromArchive,
-    getDownloadedFilesByType: getDownloadedFilesByType,
-    createContainerZipFile: createContainerZipFile,
-    durationStringToNumber: durationStringToNumber,
-    getMatchingCategoryFiles: getMatchingCategoryFiles,
-    getCurrentDownloader: getCurrentDownloader,
-    recFindByExt: recFindByExt,
-    removeFileExtension: removeFileExtension,
-    formatDateString: formatDateString,
-    cropFile: cropFile,
-    createEdgeNGrams: createEdgeNGrams,
-    wait: wait,
-    checkExistsWithTimeout: checkExistsWithTimeout,
-    fetchFile: fetchFile,
-    restartServer: restartServer,
-    injectArgs: injectArgs,
-    filterArgs: filterArgs,
-    searchObjectByString: searchObjectByString,
-    stripPropertiesFromObject: stripPropertiesFromObject,
-    getArchiveFolder: getArchiveFolder,
-    File: File
-}

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.3"
+appVersion: "4.3.1"

chart/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018 Isaac Grynsztein
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

chart/README.md

@@ -0,0 +1,3 @@
+The official YoutubeDL-Material chart
+
+Repo: https://github.com/Tzahi12345/YoutubeDL-Material

chart/artifacthub-repo.yml

@@ -0,0 +1,4 @@
+# Artifact Hub repository metadata file
+repositoryID: ff79ae03-57c1-4f18-8368-5e085d06e2f1
+owners:
+  - name: tzahi12345

chart/index.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+entries:
+  youtubedl-material:
+  - apiVersion: v1
+    created: "2021-01-01T05:51:36.304331-05:00"
+    description: A Material Design frontend for youtube-dl
+    digest: f7340d24fb051ade30b890db3b5c483a884b8459316dcf2ffc6fb9413d41252e
+    home: https://github.com/Tzahi12345/YoutubeDL-Material/
+    icon: https://i.imgur.com/IKOlr0N.png
+    keywords:
+    - youtubedl-material
+    - youtube-dl
+    maintainers:
+    - email: IsaacMGrynsztein@gmail.com
+      name: tzahi12345
+    name: youtubedl-material
+    sources:
+    - https://github.com/Tzahi12345/YoutubeDL-Material/
+    urls:
+    - youtubedl-material-0.0.1.tgz
+    version: 0.0.1
+generated: "2021-01-01T05:51:36.3003569-05:00"

chart/templates/claim-appdata-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-appdata
+  name: ytdl-material-claim-appdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "1Gi" .Values.appdataClaimSize }}
+status: {}

chart/templates/claim-audio-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-audio
+  name: ytdl-material-claim-audio
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.audioClaimSize }}
+status: {}

chart/templates/claim-subscriptions-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-subscriptions
+  name: ytdl-material-claim-subscriptions
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.subscriptionsClaimSize }}
+status: {}

chart/templates/claim-users-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-users
+  name: ytdl-material-claim-users
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.usersClaimSize }}
+status: {}

chart/templates/claim-video-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-video
+  name: ytdl-material-claim-video
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.videoClaimSize }}
+status: {}

docker-compose.yml

@@ -18,7 +18,7 @@ services:
             - "8998:17442"
         image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        image: mongo
+        image: mongo:4
         logging:
             driver: "none"          
         container_name: mongo-db

docker-utils/GetTwitchDownloader.py

@@ -0,0 +1,53 @@
+import platform
+import requests
+import shutil
+import os
+import re
+
+from github import Github
+
+machine = platform.machine()
+
+def isARM():
+    return True if machine.startswith('arm') else False
+
+def getLatestFileInRepo(repo, search_string):
+    # Create an unauthenticated instance of the Github object
+    g = Github(os.environ.get('GH_TOKEN'))
+
+    # Replace with the repository owner and name
+    repo = g.get_repo(repo)
+
+    # Get all releases of the repository
+    releases = repo.get_releases()
+
+    # Loop through the releases in reverse order (from latest to oldest)
+    for release in list(releases):
+        # Get the release assets (files attached to the release)
+        assets = release.get_assets()
+
+        # Loop through the assets
+        for asset in assets:
+            if re.search(search_string, asset.name):
+                print(f'Downloading: {asset.name}')
+                response = requests.get(asset.browser_download_url)
+                with open(asset.name, 'wb') as f:
+                    f.write(response.content)
+                print(f'Download complete: {asset.name}. Unzipping...')
+                shutil.unpack_archive(asset.name, './')
+                print(f'Unzipping complete!')
+                os.remove(asset.name)
+                break
+        else:
+            continue
+        break
+    else:
+        # If no matching release is found, print a message
+        print(f'No release found with {search_string}')
+
+def getLatestCLIRelease():
+    isArm = isARM()
+    searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
+    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
+
+getLatestCLIRelease()

docker-utils/ffmpeg-fetch.sh

@@ -26,7 +26,7 @@ apt-get update && apt-get -y install curl xz-utils
 echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
 curl -o ffmpeg.txz \
     --connect-timeout 5 \
-    --max-time 10 \
+    --max-time 120 \
     --retry 5 \
     --retry-delay 0 \
     --retry-max-time 40 \

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.3.0",
+  "version": "4.3.1",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -21,62 +21,62 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
-    "@angular/animations": "^13.3.4",
-    "@angular/cdk": "^13.3.4",
-    "@angular/common": "^13.3.4",
-    "@angular/compiler": "^13.3.4",
-    "@angular/core": "^13.3.4",
-    "@angular/forms": "^13.3.4",
-    "@angular/localize": "^13.3.4",
-    "@angular/material": "^13.3.4",
-    "@angular/platform-browser": "^13.3.4",
-    "@angular/platform-browser-dynamic": "^13.3.4",
-    "@angular/router": "^13.3.4",
+    "@angular-devkit/core": "^15.0.1",
+    "@angular/animations": "^15.0.1",
+    "@angular/cdk": "^15.0.0",
+    "@angular/common": "^15.0.1",
+    "@angular/compiler": "^15.0.1",
+    "@angular/core": "^15.0.1",
+    "@angular/forms": "^15.0.1",
+    "@angular/localize": "^15.0.1",
+    "@angular/material": "^15.0.0",
+    "@angular/platform-browser": "^15.0.1",
+    "@angular/platform-browser-dynamic": "^15.0.1",
+    "@angular/router": "^15.0.1",
     "@fontsource/material-icons": "^4.5.4",
     "@ngneat/content-loader": "^5.0.0",
-    "@videogular/ngx-videogular": "^5.0.1",
+    "@videogular/ngx-videogular": "^6.0.0",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
-    "filesize": "^6.1.0",
+    "filesize": "^10.0.7",
     "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
-    "ng-lazyload-image": "^7.0.1",
-    "ngx-avatars": "^1.3.1",
+    "ngx-avatars": "^1.4.1",
     "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
-    "rxjs-compat": "^6.0.0-rc.0",
+    "rxjs-compat": "^6.6.7",
     "tslib": "^2.0.0",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "xliff-to-json": "^1.0.4",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
-    "@angular/cli": "^13.3.3",
-    "@angular/compiler-cli": "^13.3.4",
-    "@angular/language-service": "^13.3.4",
+    "@angular-devkit/build-angular": "^15.0.1",
+    "@angular/cli": "^15.0.1",
+    "@angular/compiler-cli": "^15.0.1",
+    "@angular/language-service": "^15.0.1",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "~3.6.0",
+    "@types/jasmine": "^4.3.1",
     "@types/node": "^12.11.1",
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
+    "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
-    "electron": "^19.0.6",
+    "electron": "^19.1.9",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.3.16",
+    "karma": "~6.4.2",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
-    "karma-jasmine": "~4.0.0",
+    "karma-jasmine": "~5.1.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.21.0",
+    "openapi-typescript-codegen": "^0.23.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
 export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
@@ -26,8 +27,10 @@ export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
 export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
 export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
 export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -50,6 +53,8 @@ export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetArchivesRequest } from './models/GetArchivesRequest';
+export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -63,6 +68,7 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
 export type { GetLogsResponse } from './models/GetLogsResponse';
 export type { GetMp3sResponse } from './models/GetMp3sResponse';
 export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
 export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
 export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
 export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -73,16 +79,22 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
 export type { LoginResponse } from './models/LoginResponse';
+export type { Notification } from './models/Notification';
+export { NotificationAction } from './models/NotificationAction';
+export { NotificationType } from './models/NotificationType';
 export type { Playlist } from './models/Playlist';
 export type { RegisterRequest } from './models/RegisterRequest';
 export type { RegisterResponse } from './models/RegisterResponse';
+export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
 export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
 export type { SharingToggle } from './models/SharingToggle';
 export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
@@ -108,8 +120,10 @@ export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';
 export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
+export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/Archive.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type Archive = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    title: string;
+    user_uid?: string;
+    sub_id?: string;
+    timestamp: number;
+    uid: string;
+};

src/api-types/models/DBInfoResponse.ts

@@ -14,5 +14,6 @@ subscriptions?: TableInfo;
 users?: TableInfo;
 roles?: TableInfo;
 download_queue?: TableInfo;
+archives?: TableInfo;
 };
 };

src/api-types/models/DatabaseFile.ts

@@ -26,6 +26,7 @@ export type DatabaseFile = {
     path: string;
     upload_date: string;
     uid: string;
+    user_uid?: string;
     sharingEnabled?: boolean;
     category?: Category;
     view_count?: number;
@@ -40,4 +41,5 @@ export type DatabaseFile = {
      * In Kbps
      */
     abr?: number;
+    favorite: boolean;
 };

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type DeleteArchiveItemsRequest = {
+    archives: Array<Archive>;
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteNotificationRequest = {
+    uid: string;
+};

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,12 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { SubscriptionRequestData } from './SubscriptionRequestData';
-
 export type DeleteSubscriptionFileRequest = {
-    file: string;
-    file_uid?: string;
-    sub: SubscriptionRequestData;
+    file_uid: string;
     /**
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */

src/api-types/models/Download.ts

@@ -19,6 +19,10 @@ export type Download = {
      * Error text, set if download fails.
      */
     error?: string | null;
+    /**
+     * Error type, may or may not be set in case of an error
+     */
+    error_type?: string | null;
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,8 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { FileType } from './FileType';
+
 export type DownloadArchiveRequest = {
-    sub: {
-archive_dir: string;
-};
+    type?: FileType;
+    sub_id?: string;
 };

src/api-types/models/DownloadRequest.ts

@@ -35,10 +35,18 @@ export type DownloadRequest = {
      * Height of the video, if known
      */
     selectedHeight?: string;
+    /**
+     * Max height that should be used, useful for playlists. selectedHeight will override this.
+     */
+    maxHeight?: string;
     /**
      * Specify ffmpeg/avconv audio quality
      */
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
+    /**
+     * If using youtube-dl archive, download will ignore it
+     */
+    ignoreArchive?: boolean;
 };

src/api-types/models/GetAllFilesRequest.ts

@@ -13,6 +13,10 @@ export type GetAllFilesRequest = {
      */
     text_search?: string;
     file_type_filter?: FileTypeFilter;
+    /**
+     * If set to true, only gets favorites
+     */
+    favorite_filter?: boolean;
     /**
      * Include if you want to filter by subscription
      */

src/api-types/models/GetArchivesRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type GetArchivesRequest = {
+    type?: FileType;
+    sub_id?: string;
+};

src/api-types/models/GetArchivesResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type GetArchivesResponse = {
+    archives: Array<Archive>;
+};

src/api-types/models/GetNotificationsResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Notification } from './Notification';
+
+export type GetNotificationsResponse = {
+    notifications?: Array<Notification>;
+};

src/api-types/models/ImportArchiveRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type ImportArchiveRequest = {
+    archive: string;
+    type: FileType;
+    sub_id?: string;
+};

src/api-types/models/Notification.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { NotificationAction } from './NotificationAction';
+import type { NotificationType } from './NotificationType';
+
+export type Notification = {
+    type: NotificationType;
+    uid: string;
+    user_uid?: string;
+    action?: Array<NotificationAction>;
+    read: boolean;
+    data?: any;
+    timestamp: number;
+};

src/api-types/models/NotificationAction.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationAction {
+    PLAY = 'play',
+    RETRY_DOWNLOAD = 'retry_download',
+    VIEW_DOWNLOAD_ERROR = 'view_download_error',
+    VIEW_TASKS = 'view_tasks',
+}

src/api-types/models/NotificationType.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationType {
+    DOWNLOAD_COMPLETE = 'download_complete',
+    DOWNLOAD_ERROR = 'download_error',
+    TASK_FINISHED = 'task_finished',
+}

src/api-types/models/Playlist.ts

@@ -14,4 +14,5 @@ export type Playlist = {
     duration: number;
     user_uid?: string;
     auto?: boolean;
+    sharingEnabled?: boolean;
 };

src/api-types/models/RestartDownloadResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { SuccessObject } from './SuccessObject';
+
+export type RestartDownloadResponse = (SuccessObject & {
+new_download_uid?: string;
+});

src/api-types/models/Schedule.ts

@@ -9,6 +9,7 @@ dayOfWeek?: Array<number>;
 hour?: number;
 minute?: number;
 timestamp?: number;
+tz?: string;
 };
 };
 

src/api-types/models/SetNotificationsToReadRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type SetNotificationsToReadRequest = {
+    uids: Array<string>;
+};

src/api-types/models/Task.ts

@@ -12,4 +12,5 @@ export type Task = {
     data: any;
     error: string;
     schedule: any;
+    options?: any;
 };

src/api-types/models/UpdateTaskOptionsRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UpdateTaskOptionsRequest = {
+    task_key: string;
+    new_options: any;
+};

src/api-types/models/UploadCookiesRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UploadCookiesRequest = {
+    cookies: Blob;
+};

src/app/app-routing.module.ts

@@ -23,7 +23,7 @@ const routes: Routes = [
 ];
 
 @NgModule({
-  imports: [RouterModule.forRoot(routes, { useHash: true, relativeLinkResolution: 'legacy' })],
+  imports: [RouterModule.forRoot(routes, { useHash: true })],
   exports: [RouterModule]
 })
 export class AppRoutingModule { }

src/app/app.component.css

@@ -13,7 +13,6 @@
 }
 
 .theme-slide-toggle {
-    top: 2px;
     left: 10px;
     position: relative;
     pointer-events: none;
@@ -25,4 +24,20 @@
 
 .top-toolbar {
     height: 64px;
-}
+    background: unset;
+}
+
+::ng-deep .top-menu-button > span {
+    width: 85px;
+    height: 26px;
+}
+
+::ng-deep .mdc-switch {
+    outline: none !important;
+}
+
+::ng-deep .notifications-menu {
+    width: 30vw !important;
+    max-width: 100% !important;
+    min-width: 280px !important;
+}

src/app/app.component.html

@@ -1,37 +1,41 @@
 <div [style.background]="postsService.theme ? postsService.theme.background_color : null" style="width: 100%; height: 100%;">
     <div class="mat-elevation-z3" style="position: relative; z-index: 10;">
-        <mat-toolbar color="primary" class="sticky-toolbar top-toolbar">
-            <div class="flex-row" width="100%" height="100%">
-                <div class="flex-column" style="text-align: left; margin-top: 1px;">
-                    <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
-                    <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
-                </div>
-                <div class="flex-column" style="text-align: center; margin-top: 5px;">
-                    <div style="font-size: 22px; text-shadow: #141414 0.25px 0.25px 1px;">
-                        {{topBarTitle}}
+        <mat-toolbar class="sticky-toolbar top-toolbar">
+            <div class="container-fluid" style="padding-left: 0px; padding-right: 0px">
+                <div class="row" width="100%" height="100%">
+                    <div class="col-6" style="text-align: left; margin-top: 1px;">
+                        <div style="display: flex; align-items: center;">
+                            <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
+                            <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
+                            <div style="margin-left: 8px; display: inline-block;"><button mat-icon-button routerLink='/home'><img style="width: 32px;" src="assets/images/logo_128px.png"></button></div>
+                        </div>
+                    </div>
+                    <div class="col-6" style="text-align: right; align-items: flex-end; display: inline-block">
+                        <button *ngIf="postsService.config?.Extra.enable_notifications" [matMenuTriggerFor]="notificationsMenu" (menuOpened)="notificationMenuOpened()" mat-icon-button><mat-icon [matBadge]="notification_count" matBadgeColor="warn" matBadgeSize="small" *ngIf="notification_count > 0">notifications</mat-icon><mat-icon *ngIf="notification_count === 0">notifications_none</mat-icon></button>
+                        <mat-menu [classList]="'notifications-menu'" (close)="notificationMenuClosed()" #notificationsMenu="matMenu">
+                            <app-notifications #notifications (notificationCount)="notificationCountUpdate($event)" (click)="$event.stopPropagation()"></app-notifications>
+                        </mat-menu>
+                        <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
+                        <mat-menu #menuSettings="matMenu">
+                            <button class="top-menu-button" (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
+                                <mat-icon>person</mat-icon>
+                                <span i18n="Profile menu label">Profile</span>
+                            </button>
+                            <button *ngIf="postsService.config && postsService.config.Downloader.use_youtubedl_archive" class="top-menu-button" (click)="openArchivesDialog()" mat-menu-item>
+                                <mat-icon>topic</mat-icon>
+                                <span i18n="Archives menu label">Archives</span>
+                            </button>
+                            <button class="top-menu-button" (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
+                            <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
+                            <span i18n="Dark mode toggle label">Dark</span>
+                            <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
+                            </button>
+                            <button class="top-menu-button" (click)="openAboutDialog()" mat-menu-item>
+                                <mat-icon>info</mat-icon>
+                                <span i18n="About menu label">About</span>
+                            </button>
+                        </mat-menu>
                     </div>
-                </div>
-                <div class="flex-column" style="text-align: right; align-items: flex-end;">
-                    <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
-                    <mat-menu #menuSettings="matMenu">
-                        <button (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
-                            <mat-icon>person</mat-icon>
-                            <span i18n="Profile menu label">Profile</span>
-                        </button>
-                        <button (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
-                          <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
-                          <span i18n="Dark mode toggle label">Dark</span>
-                          <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
-                        </button>
-                        <!-- <button *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('settings')))" (click)="openSettingsDialog()" mat-menu-item>
-                          <mat-icon>settings</mat-icon>
-                          <span i18n="Settings menu label">Settings</span>
-                        </button> -->
-                        <button (click)="openAboutDialog()" mat-menu-item>
-                            <mat-icon>info</mat-icon>
-                            <span i18n="About menu label">About</span>
-                        </button>
-                    </mat-menu>
                 </div>
             </div>
         </mat-toolbar>
@@ -51,7 +55,7 @@
                     </ng-container>
                     <ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
                         <mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
-                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
+                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.display]="'inline-block'" [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
                     </ng-container>
                 </mat-nav-list>
             </mat-sidenav>

src/app/app.component.ts

@@ -20,6 +20,8 @@ import { SettingsComponent } from './settings/settings.component';
 import { AboutDialogComponent } from './dialogs/about-dialog/about-dialog.component';
 import { UserProfileDialogComponent } from './dialogs/user-profile-dialog/user-profile-dialog.component';
 import { SetDefaultAdminDialogComponent } from './dialogs/set-default-admin-dialog/set-default-admin-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 @Component({
   selector: 'app-root',
@@ -45,9 +47,12 @@ export class AppComponent implements OnInit, AfterViewInit {
   enableDownloadsManager = false;
 
   @ViewChild('sidenav') sidenav: MatSidenav;
+  @ViewChild('notifications') notifications: NotificationsComponent;
   @ViewChild('hamburgerMenu', { read: ElementRef }) hamburgerMenuButton: ElementRef;
   navigator: string = null;
 
+  notification_count = 0;
+
   constructor(public postsService: PostsService, public snackBar: MatSnackBar, private dialog: MatDialog,
     public router: Router, public overlayContainer: OverlayContainer, private elementRef: ElementRef) {
 
@@ -71,7 +76,7 @@ export class AppComponent implements OnInit, AfterViewInit {
 
   }
 
-  ngOnInit() {
+  ngOnInit(): void {
     if (localStorage.getItem('theme')) {
       this.setTheme(localStorage.getItem('theme'));
     }
@@ -90,15 +95,15 @@ export class AppComponent implements OnInit, AfterViewInit {
     });
   }
 
-  ngAfterViewInit() {
+  ngAfterViewInit(): void {
     this.postsService.sidenav = this.sidenav;
   }
 
-  toggleSidenav() {
+  toggleSidenav(): void {
     this.sidenav.toggle();
   }
 
-  loadConfig() {
+  loadConfig(): void {
     // loading config
     this.topBarTitle = this.postsService.config['Extra']['title_top'];
     const themingExists = this.postsService.config['Themes'];
@@ -164,7 +169,7 @@ export class AppComponent implements OnInit, AfterViewInit {
     this.componentCssClass = theme;
   }
 
-  flipTheme() {
+  flipTheme(): void {
     if (this.postsService.theme.key === 'default') {
       this.setTheme('dark');
     } else if (this.postsService.theme.key === 'dark') {
@@ -172,17 +177,12 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  themeMenuItemClicked(event) {
+  themeMenuItemClicked(event): void {
     this.flipTheme();
     event.stopPropagation();
   }
 
-  getSubscriptions() {
-
-  }
-
-
-  goBack() {
+  goBack(): void {
     if (!this.navigator) {
       this.router.navigate(['/home']);
     } else {
@@ -190,23 +190,41 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  openSettingsDialog() {
-    const dialogRef = this.dialog.open(SettingsComponent, {
+  openSettingsDialog(): void {
+    this.dialog.open(SettingsComponent, {
       width: '80vw'
     });
   }
 
-  openAboutDialog() {
-    const dialogRef = this.dialog.open(AboutDialogComponent, {
+  openAboutDialog(): void {
+    this.dialog.open(AboutDialogComponent, {
       width: '80vw'
     });
   }
 
-  openProfileDialog() {
-    const dialogRef = this.dialog.open(UserProfileDialogComponent, {
+  openProfileDialog(): void {
+    this.dialog.open(UserProfileDialogComponent, {
       width: '60vw'
     });
   }
 
+  openArchivesDialog(): void {
+    this.dialog.open(ArchiveViewerComponent, {
+      width: '85vw'
+    });
+  }
+
+  notificationCountUpdate(new_count: number): void {
+    this.notification_count = new_count;
+  }
+
+  notificationMenuOpened(): void {
+    this.notifications.getNotifications();
+  }
+
+  notificationMenuClosed(): void {
+    this.notifications.setNotificationsToRead();
+  }
+
 }
 

src/app/app.module.ts

@@ -29,6 +29,8 @@ import { MatPaginatorModule } from '@angular/material/paginator';
 import { MatSortModule } from '@angular/material/sort';
 import { MatTableModule } from '@angular/material/table';
 import { MatDatepickerModule } from '@angular/material/datepicker';
+import { MatChipsModule } from '@angular/material/chips';
+import { MatBadgeModule } from '@angular/material/badge';
 import { DragDropModule } from '@angular/cdk/drag-drop';
 import { ClipboardModule } from '@angular/cdk/clipboard';
 import { TextFieldModule } from '@angular/cdk/text-field';
@@ -51,7 +53,6 @@ import { SubscribeDialogComponent } from './dialogs/subscribe-dialog/subscribe-d
 import { SubscriptionComponent } from './subscription//subscription/subscription.component';
 import { SubscriptionInfoDialogComponent } from './dialogs/subscription-info-dialog/subscription-info-dialog.component';
 import { SettingsComponent } from './settings/settings.component';
-import { MatChipsModule } from '@angular/material/chips';
 import { NgxFileDropModule } from 'ngx-file-drop';
 import { AvatarModule } from 'ngx-avatars';
 import { ContentLoaderModule } from '@ngneat/content-loader';
@@ -87,6 +88,13 @@ import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-butto
 import { TasksComponent } from './components/tasks/tasks.component';
 import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
 import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { NotificationsListComponent } from './components/notifications-list/notifications-list.component';
+import { TaskSettingsComponent } from './components/task-settings/task-settings.component';
+import { GenerateRssUrlComponent } from './dialogs/generate-rss-url/generate-rss-url.component';
+import { SortPropertyComponent } from './components/sort-property/sort-property.component';
+import { OnlyNumberDirective } from './directives/only-number.directive';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 registerLocaleData(es, 'es');
 
@@ -132,7 +140,14 @@ registerLocaleData(es, 'es');
         SkipAdButtonComponent,
         TasksComponent,
         UpdateTaskScheduleDialogComponent,
-        RestoreDbDialogComponent
+        RestoreDbDialogComponent,
+        NotificationsComponent,
+        NotificationsListComponent,
+        TaskSettingsComponent,
+        GenerateRssUrlComponent,
+        SortPropertyComponent,
+        OnlyNumberDirective,
+        ArchiveViewerComponent
     ],
     imports: [
         CommonModule,
@@ -170,6 +185,7 @@ registerLocaleData(es, 'es');
         MatTableModule,
         MatDatepickerModule,
         MatChipsModule,
+        MatBadgeModule,
         DragDropModule,
         ClipboardModule,
         TextFieldModule,

src/app/components/archive-viewer/archive-viewer.component.html

@@ -0,0 +1,143 @@
+<mat-form-field class="filter">
+  <mat-icon matPrefix>search</mat-icon>
+  <mat-label i18n="Filter">Filter</mat-label>
+  <input matInput [(ngModel)]="text_filter" (keyup)="applyFilter($event)" #input>
+</mat-form-field>
+
+<div [hidden]="!(archives && archives.length > 0)">
+    <div class="mat-elevation-z8">
+        <mat-table matSort [dataSource]="dataSource">
+
+          <!-- Select Column -->
+            <!-- Checkbox Column -->
+            <ng-container matColumnDef="select">
+              <mat-header-cell *matHeaderCellDef>
+                <mat-checkbox (change)="$event ? toggleAllRows() : null"
+                              [checked]="selection.hasValue() && isAllSelected()"
+                              [indeterminate]="selection.hasValue() && !isAllSelected()">
+                </mat-checkbox>
+              </mat-header-cell>
+              <mat-cell *matCellDef="let row">
+                <mat-checkbox (click)="$event.stopPropagation()"
+                              (change)="$event ? selection.toggle(row) : null"
+                              [checked]="selection.isSelected(row)">
+                </mat-checkbox>
+                <mat-icon class="audio-video-icon">{{(row.type === 'audio') ? 'audiotrack' : 'movie'}}</mat-icon>
+              </mat-cell>
+            </ng-container>
+      
+          <!-- Date Column -->
+          <ng-container matColumnDef="timestamp">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Date">Date</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element"> {{element.timestamp*1000 | date: 'short'}} </mat-cell>
+          </ng-container>
+      
+          <!-- Title Column -->
+          <ng-container matColumnDef="title">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="max-two-lines" [matTooltip]="element.title ? element.title : null">
+                    {{element.title}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+           <!-- ID Column -->
+           <ng-container matColumnDef="id">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="ID">ID</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.title ? element.title : null">
+                    {{element.id}}
+                </span>
+            </mat-cell>
+          </ng-container>
+
+          <!-- Extractor Column -->
+          <ng-container matColumnDef="extractor">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Extractor">Extractor</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.extractor? element.extractor : null">
+                    {{element.extractor}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+            <mat-header-row *matHeaderRowDef="displayedColumns; sticky: true"></mat-header-row>
+            <mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
+        </mat-table>
+      </div>
+</div>
+
+<div *ngIf="(!archives || archives.length === 0)">
+  <h4 style="text-align: center; margin-top: 10px;" i18n="Archives empty">Archives empty</h4>
+</div>
+
+<div style="margin: 10px 10px 10px 0px; display: flex;">
+  <span style="flex-grow: 1;" class="flex-items">
+    <button [disabled]="selection.selected.length === 0" color="warn" style="margin: 10px;" mat-stroked-button i18n="Delete selected" (click)="openDeleteSelectedArchivesDialog()">Delete selected</button>
+  </span>
+  <span class="flex-items">
+    <button [disabled]="!(archives && archives.length > 0)" (click)="downloadArchive()" mat-stroked-button i18n="Download archive">Download archive</button>
+    <mat-form-field style="width: 150px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="Subscription">Subscription</mat-label>
+      <mat-select [ngModel]="sub_id" (ngModelChange)="subFilterSelectionChanged($event)">
+        <mat-option [value]="'none'" i18n="None">None</mat-option>
+        <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+      </mat-select>
+    </mat-form-field>
+    <mat-form-field style="width: 100px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="File type">File type</mat-label>
+      <mat-select [ngModel]="type" (ngModelChange)="typeFilterSelectionChanged($event)" [disabled]="sub_id !== 'none'">
+        <mat-option [value]="'both'" i18n="Both">Both</mat-option>
+        <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+        <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+      </mat-select>
+    </mat-form-field>
+  </span>
+</div>
+
+<div class="file-drop-parent">
+  <ngx-file-drop [multiple]="false" accept=".txt" dropZoneLabel="Drop file here" (onFileDrop)="dropped($event)">
+      <ng-template class="file-drop" ngx-file-drop-content-tmp let-openFileSelector="openFileSelector">
+          <div style="text-align: center">
+              <div>
+                  <ng-container i18n="Drag and Drop">Drag and Drop</ng-container>
+              </div>
+              <div style="margin-top: 6px;">
+                  <button mat-stroked-button (click)="openFileSelector()">Browse Files</button>
+              </div>
+          </div>
+      </ng-template>
+  </ngx-file-drop>
+</div>
+
+<div style="margin-top: 10px; color: white">
+  <table class="table">
+      <tbody class="upload-name-style">
+          <tr *ngFor="let item of files; let i=index">
+              <td style="vertical-align: middle; border-top: unset">
+                  <strong>{{ item.relativePath }}</strong>
+              </td>
+              <td style="border-top: unset">
+                <div style="float: right">
+                  <mat-form-field style="width: 150px;">
+                    <mat-label i18n="Subscription">Subscription</mat-label>
+                    <mat-select [ngModel]="upload_sub_id" (ngModelChange)="subUploadFilterSelectionChanged($event)">
+                      <mat-option [value]="'none'" i18n="None">None</mat-option>
+                      <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <mat-form-field style="width: 100px; margin-left: 10px">
+                    <mat-label i18n="File type">File type</mat-label>
+                    <mat-select [(ngModel)]="upload_type" [value]="upload_type" [disabled]="upload_sub_id !== 'none'">
+                      <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+                      <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <button style="margin-left: 10px" [disabled]="uploading_archive || uploaded_archive" (click)="importArchive()" matTooltip="Upload" i18n-matTooltip="Upload" mat-mini-fab><mat-icon>publish</mat-icon><mat-spinner *ngIf="uploading_archive" class="spinner" [diameter]="38"></mat-spinner></button>
+                </div>
+              </td>
+          </tr>
+      </tbody>
+  </table>
+</div>

src/app/components/archive-viewer/archive-viewer.component.scss

@@ -0,0 +1,42 @@
+.filter {
+    width: 100%;
+}
+
+.spinner {
+    bottom: 1px;
+    left: 0.5px;
+    position: absolute;
+}
+
+.mat-mdc-table {
+    width: 100%;
+    max-height: 60vh;
+    overflow: auto;
+}
+
+.max-two-lines {
+    display: -webkit-box;
+    display: -moz-box;
+    max-height: 2.4em;
+    line-height: 1.2em;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    -webkit-box-orient: vertical;
+    -webkit-line-clamp: 2;
+}
+
+:host ::ng-deep .ngx-file-drop__content {
+    width: 100%;
+    top: -12px;
+    position: relative;
+}
+
+.file-drop-parent {
+    padding: 0px 10px 0px 10px;
+}
+
+.flex-items {
+    display: flex;
+    align-items: center;
+}
+

src/app/components/archive-viewer/archive-viewer.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ArchiveViewerComponent } from './archive-viewer.component';
+
+describe('ArchiveViewerComponent', () => {
+  let component: ArchiveViewerComponent;
+  let fixture: ComponentFixture<ArchiveViewerComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ArchiveViewerComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(ArchiveViewerComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/archive-viewer/archive-viewer.component.ts

@@ -0,0 +1,198 @@
+import { SelectionModel } from '@angular/cdk/collections';
+import { Component, ViewChild } from '@angular/core';
+import { MatDialog } from '@angular/material/dialog';
+import { MatSort } from '@angular/material/sort';
+import { MatTableDataSource } from '@angular/material/table';
+import { FileType } from 'api-types';
+import { Archive } from 'api-types/models/Archive';
+import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
+import { PostsService } from 'app/posts.services';
+import { NgxFileDropEntry } from 'ngx-file-drop';
+
+@Component({
+  selector: 'app-archive-viewer',
+  templateUrl: './archive-viewer.component.html',
+  styleUrls: ['./archive-viewer.component.scss']
+})
+export class ArchiveViewerComponent {
+  // table
+  displayedColumns: string[] = ['select', 'timestamp', 'title', 'id', 'extractor'];
+  dataSource = null;
+  selection = new SelectionModel<Archive>(true, []);
+
+  // general
+  archives = null;
+  archives_retrieved = false;
+  text_filter = '';
+  sub_id = 'none';
+  upload_sub_id = 'none';
+  type: FileType | 'both' = 'both';
+  upload_type: FileType = FileType.VIDEO;
+
+  // importing
+  uploading_archive = false;
+  uploaded_archive = false;
+  files = [];
+
+  typeSelectOptions = {
+    video: {
+      key: 'video',
+      label: $localize`Video`
+    },
+    audio: {
+      key: 'audio',
+      label: $localize`Audio`
+    }
+  };
+
+  @ViewChild(MatSort) sort: MatSort;
+  
+  constructor(public postsService: PostsService, private dialog: MatDialog) {
+
+  }
+
+  ngOnInit() {
+    this.getArchives();
+  }
+
+  applyFilter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.dataSource.filter = filterValue.trim().toLowerCase();
+  }
+
+  /** Whether the number of selected elements matches the total number of rows. */
+  isAllSelected() {
+    const numSelected = this.selection.selected.length;
+    const numRows = this.dataSource.data.length;
+    return numSelected === numRows;
+  }
+
+  /** Selects all rows if they are not all selected; otherwise clear selection. */
+  toggleAllRows() {
+    if (this.isAllSelected()) {
+      this.selection.clear();
+      return;
+    }
+
+    this.selection.select(...this.dataSource.data);
+  }
+
+  typeFilterSelectionChanged(value): void {
+    this.type = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    this.getArchives();
+  }
+
+  subFilterSelectionChanged(value): void {
+    this.sub_id = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    if (this.sub_id !== 'none') {
+      this.type = this.postsService.getSubscriptionByID(this.sub_id)['type'];
+    }
+    this.getArchives();
+  }
+
+  subUploadFilterSelectionChanged(value): void {
+    this.upload_sub_id = value;
+    if (this.upload_sub_id !== 'none') {
+      this.upload_type = this.postsService.getSubscriptionByID(this.upload_sub_id)['type'];
+    }
+  }
+
+  getArchives(): void {
+    this.postsService.getArchives(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      if (res['archives'] !== null 
+        && res['archives'] !== undefined
+        && JSON.stringify(this.archives) !== JSON.stringify(res['archives'])) {
+          this.archives = res['archives']
+          this.dataSource = new MatTableDataSource<Archive>(this.archives);
+          this.dataSource.sort = this.sort;
+      } else {
+        // failed to get downloads
+      }
+    });
+  }
+
+  importArchive(): void {
+    this.uploading_archive = true;
+        for (const droppedFile of this.files) {
+      // Is it a file?
+      if (droppedFile.fileEntry.isFile) {
+        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
+        fileEntry.file(async (file: File) => {          
+          const archive_base64 = await blobToBase64(file);
+          this.postsService.importArchive(archive_base64 as string, this.upload_type, this.upload_sub_id === 'none' ? null : this.upload_sub_id).subscribe(res => {
+            this.uploading_archive = false;
+            if (res['success']) {
+              this.uploaded_archive = true;
+              this.postsService.openSnackBar($localize`Archive successfully imported!`);
+            }
+            this.getArchives();
+          }, err => {
+            console.error(err);
+            this.uploading_archive = false;
+          });
+        });
+      }
+    }
+  }
+
+  downloadArchive(): void {
+    this.postsService.downloadArchive(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      const blob: Blob = res;
+      saveAs(blob, 'archive.txt');
+    });
+  }
+
+  openDeleteSelectedArchivesDialog(): void {
+    const dialogRef = this.dialog.open(ConfirmDialogComponent, {
+      data: {
+        dialogTitle: $localize`Delete archives`,
+        dialogText: $localize`Would you like to delete ${this.selection.selected.length}:selected archives amount: archive(s)?`,
+        submitText: $localize`Delete`,
+        warnSubmitColor: true
+      }
+    });
+    dialogRef.afterClosed().subscribe(confirmed => {
+      if (confirmed) {
+        this.deleteSelectedArchives();
+      }
+    });
+    
+  }
+
+  deleteSelectedArchives(): void {
+    for (const archive of this.selection.selected) {
+      this.archives = this.archives.filter((_archive: Archive) => !(archive['extractor'] === _archive['extractor'] && archive['id'] !== _archive['id']));
+    }
+    this.postsService.deleteArchiveItems(this.selection.selected).subscribe(res => {
+      if (res['success']) {
+        this.postsService.openSnackBar($localize`Successfully deleted archive items!`);
+      } else {
+        this.postsService.openSnackBar($localize`Failed to delete archive items!`);
+      }
+      this.getArchives();
+    });
+    this.selection.clear();
+  }
+
+  public dropped(files: NgxFileDropEntry[]) {
+    this.files = files;
+    this.uploading_archive = false;
+    this.uploaded_archive = false;
+  }
+
+  originalOrder = (): number => {
+    return 0;
+  }
+}
+
+function blobToBase64(blob: Blob) {
+  return new Promise((resolve, _) => {
+    const reader = new FileReader();
+    reader.onloadend = () => resolve(reader.result);
+    reader.readAsDataURL(blob);
+  });
+}

src/app/components/custom-playlists/custom-playlists.component.html

@@ -2,7 +2,7 @@
     <div class="container">
         <div class="row justify-content-center">
             <div *ngFor="let playlist of playlists; let i = index" class="mb-2 mt-2" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
-                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [loading]="false"></app-unified-file-card>
+                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? this.postsService.token : ''" [loading]="false"></app-unified-file-card>
             </div>
         </div>
     </div>

src/app/components/downloads/downloads.component.ts

@@ -41,7 +41,7 @@ import { Download } from 'api-types';
 })
 export class DownloadsComponent implements OnInit, OnDestroy {
 
-  @Input() uids = null;
+  @Input() uids: string[] = null;
 
   downloads_check_interval = 1000;
   downloads = [];
@@ -200,6 +200,10 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     this.postsService.restartDownload(download_uid).subscribe(res => {
       if (!res['success']) {
         this.postsService.openSnackBar($localize`Failed to restart download! See server logs for more info.`);
+      } else {
+        if (this.uids && res['new_download_uid']) {
+          this.uids.push(res['new_download_uid']);
+        }
       }
     });
   }

src/app/components/login/login.component.html

@@ -1,31 +1,36 @@
 <mat-card class="login-card">
-    <mat-tab-group style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
+    <mat-tab-group mat-stretch-tabs style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
         <mat-tab label="Login" i18n-label="Login">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="loginUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="loginUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>
         <mat-tab *ngIf="registrationEnabled" label="Register" i18n-label="Register">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="registrationUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="registrationUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput placeholder="Confirm Password" i18n-placeholder="Confirm Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Confirm Password">Confirm Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>

src/app/components/login/login.component.scss

@@ -12,17 +12,15 @@
 }
 
 .login-button-div {
-    margin-bottom: 10px;
-    margin-top: 10px;
-    margin-left: -16px;
-    margin-right: -16px;
     bottom: 0px;
     width: 100%;
+    height: 42px;
     position: absolute;
 }
 
 .login-button-div > button {
     width: 100%;
+    height: 100%;
     border-radius: 0px 0px 4px 4px !important;
 }
 

src/app/components/manage-role/manage-role.component.html

@@ -1,17 +1,15 @@
-<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.name}}</h4>
+<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.key}}</h4>
 
 <mat-dialog-content *ngIf="role">
-    <mat-list>
-        <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
-            <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
-            <span matLine>
-                <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
-                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
-                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
-                </mat-radio-group>
-            </span>
-        </mat-list-item>
-    </mat-list>
+    <div *ngFor="let permission of available_permissions">
+        <div matListItemTitle>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</div>
+        <div matListItemLine>
+            <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
+                <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
+                <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
+            </mat-radio-group>
+        </div>
+    </div>
 </mat-dialog-content>
 
 <mat-dialog-actions>

src/app/components/manage-role/manage-role.component.scss

@@ -1,4 +0,0 @@
-.mat-radio-button {
-    margin-right: 10px;
-    margin-top: 5px;
-}

src/app/components/manage-role/manage-role.component.ts

@@ -24,7 +24,7 @@ export class ManageRoleComponent implements OnInit {
   }
 
   constructor(public postsService: PostsService, private dialogRef: MatDialogRef<ManageRoleComponent>,
-              @Inject(MAT_DIALOG_DATA) public data: any) {
+              @Inject(MAT_DIALOG_DATA) public data: {role: string}) {
     if (this.data) {
       this.role = this.data.role;
       this.available_permissions = this.postsService.available_permissions;

src/app/components/manage-user/manage-user.component.html

@@ -5,24 +5,23 @@
 
     <div>
         <mat-form-field style="margin-right: 15px;">
-            <input matInput [(ngModel)]="newPasswordInput" type="password" placeholder="New password" i18n-placeholder="New password placeholder">
+            <mat-label i18n="New password">New password</mat-label>
+            <input matInput [(ngModel)]="newPasswordInput" type="password">
         </mat-form-field>
         <button mat-raised-button color="accent" (click)="setNewPassword()" [disabled]="newPasswordInput.length === 0"><ng-container i18n="Set new password">Set new password</ng-container></button>
     </div>
 
     <div>
-        <mat-list>
-            <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
-                <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
-                <span matLine>
-                    <mat-radio-group [disabled]="permission === 'settings' && postsService.user.uid === user.uid" (change)="changeUserPermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give user permission for ' + permission">
-                        <mat-radio-button value="default"><ng-container i18n="Use role default">Use role default</ng-container></mat-radio-button>
-                        <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
-                        <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
-                    </mat-radio-group>
-                </span>
-            </mat-list-item>
-        </mat-list>
+        <div *ngFor="let permission of available_permissions">
+            <div matListItemTitle>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</div>
+            <div matListItemLine>
+                <mat-radio-group [disabled]="permission === 'settings' && postsService.user.uid === user.uid" (change)="changeUserPermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give user permission for ' + permission">
+                    <mat-radio-button value="default"><ng-container i18n="Use role default">Use role default</ng-container></mat-radio-button>
+                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
+                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
+                </mat-radio-group>
+            </div>
+        </div>
     </div>
 </mat-dialog-content>
 

src/app/components/manage-user/manage-user.component.scss

@@ -1,4 +1,4 @@
-.mat-radio-button {
+.mat-mdc-radio-button {
     margin-right: 10px;
     margin-top: 5px;
 }

src/app/components/manage-user/manage-user.component.ts

@@ -1,6 +1,7 @@
 import { Component, OnInit, Inject } from '@angular/core';
 import { PostsService } from 'app/posts.services';
 import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { User } from 'api-types';
 
 @Component({
   selector: 'app-manage-user',
@@ -15,17 +16,18 @@ export class ManageUserComponent implements OnInit {
   permissions = null;
 
   permissionToLabel = {
-    'filemanager': 'File manager',
-    'settings': 'Settings access',
-    'subscriptions': 'Subscriptions',
-    'sharing': 'Share files',
-    'advanced_download': 'Use advanced download mode',
-    'downloads_manager': 'Use downloads manager'
+    'filemanager': $localize`File manager`,
+    'settings': $localize`Settings access`,
+    'subscriptions': $localize`Subscriptions`,
+    'sharing': $localize`Share files`,
+    'advanced_download': $localize`Use advanced download mode`,
+    'downloads_manager': $localize`Use downloads manager`,
+    'tasks_manager': $localize`Use tasks manager`,
   }
 
   settingNewPassword = false;
 
-  constructor(public postsService: PostsService, @Inject(MAT_DIALOG_DATA) public data: any) {
+  constructor(public postsService: PostsService, @Inject(MAT_DIALOG_DATA) public data: {user: User}) {
     if (this.data) {
       this.user = this.data.user;
       this.available_permissions = this.postsService.available_permissions;
@@ -53,14 +55,14 @@ export class ManageUserComponent implements OnInit {
   }
 
   changeUserPermissions(change, permission) {
-    this.postsService.setUserPermission(this.user.uid, permission, change.value).subscribe(res => {
+    this.postsService.setUserPermission(this.user.uid, permission, change.value).subscribe(() => {
       // console.log(res);
     });
   }
 
   setNewPassword() {
     this.settingNewPassword = true;
-    this.postsService.changeUserPassword(this.user.uid, this.newPasswordInput).subscribe(res => {
+    this.postsService.changeUserPassword(this.user.uid, this.newPasswordInput).subscribe(() => {
       this.newPasswordInput = '';
       this.settingNewPassword = false;
     });

src/app/components/modify-users/modify-users.component.html

@@ -1,14 +1,15 @@
 <div *ngIf="dataSource; else loading">
     <div style="padding: 15px">
     <div class="row">
-        <div class="table table-responsive pb-4 pt-2">
+        <div class="table table-responsive pb-4 pt-4">
         <div class="example-header">
-            <mat-form-field>
-            <input matInput (keyup)="applyFilter($event.target.value)" placeholder="Search" i18n-placeholder="search field description">
+            <mat-form-field appearance="outline">
+                <mat-label i18n="Search">Search</mat-label>
+                <input matInput (keyup)="applyFilter($event)">
             </mat-form-field>
         </div>
         
-        <div class="example-container mat-elevation-z8">
+        <div class="mat-elevation-z8" style="margin-right: 15px;">
         
             <mat-table #table [dataSource]="dataSource" matSort>
         

src/app/components/modify-users/modify-users.component.ts

@@ -8,6 +8,7 @@ import { MatDialog, MatDialogRef } from '@angular/material/dialog';
 import { AddUserDialogComponent } from 'app/dialogs/add-user-dialog/add-user-dialog.component';
 import { ManageUserComponent } from '../manage-user/manage-user.component';
 import { ManageRoleComponent } from '../manage-role/manage-role.component';
+import { User } from 'api-types';
 
 @Component({
   selector: 'app-modify-users',
@@ -31,7 +32,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
 
   // MatPaginator Output
   pageEvent: PageEvent;
-  users: any;
+  users: User[];
   editObject = null;
   constructedObject = {};
   roles = null;
@@ -62,7 +63,8 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     this.pageSizeOptions = setPageSizeOptionsInput.split(',').map(str => +str);
   }
 
-  applyFilter(filterValue: string) {
+  applyFilter(event: KeyboardEvent) {
+    let filterValue = (event.target as HTMLInputElement).value; // "as HTMLInputElement" is required: https://angular.io/guide/user-input#type-the-event
     filterValue = filterValue.trim(); // Remove whitespace
     filterValue = filterValue.toLowerCase(); // Datasource defaults to lowercase matches
     this.dataSource.filter = filterValue;
@@ -94,11 +96,9 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     });
   }
 
-  finishEditing(user_uid) {
-    let has_finished = false;
+  finishEditing(user_uid: string) {
     if (this.constructedObject && this.constructedObject['name'] && this.constructedObject['role']) {
       if (!isEmptyOrSpaces(this.constructedObject['name']) && !isEmptyOrSpaces(this.constructedObject['role'])) {
-        has_finished = true;
         const index_of_object = this.indexOfUser(user_uid);
         this.users[index_of_object] = this.constructedObject;
         this.constructedObject = {};
@@ -109,7 +109,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     }
   }
 
-  enableEditMode(user_uid) {
+  enableEditMode(user_uid: string) {
     if (this.uidInUserList(user_uid) && this.indexOfUser(user_uid) > -1) {
       const users_index = this.indexOfUser(user_uid);
       this.editObject = this.users[users_index];
@@ -124,7 +124,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   // checks if user is in users array by name
-  uidInUserList(user_uid) {
+  uidInUserList(user_uid: string) {
     for (let i = 0; i < this.users.length; i++) {
       if (this.users[i].uid === user_uid) {
         return true;
@@ -134,7 +134,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   // gets index of user in users array by name
-  indexOfUser(user_uid) {
+  indexOfUser(user_uid: string) {
     for (let i = 0; i < this.users.length; i++) {
       if (this.users[i].uid === user_uid) {
         return i;
@@ -144,12 +144,12 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   setUser(change_obj) {
-    this.postsService.changeUser(change_obj).subscribe(res => {
+    this.postsService.changeUser(change_obj).subscribe(() => {
       this.getArray();
     });
   }
 
-  manageUser(user_uid) {
+  manageUser(user_uid: string) {
     const index_of_object = this.indexOfUser(user_uid);
     const user_obj = this.users[index_of_object];
     this.dialog.open(ManageUserComponent, {
@@ -160,17 +160,17 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     });
   }
 
-  removeUser(user_uid) {
-    this.postsService.deleteUser(user_uid).subscribe(res => {
+  removeUser(user_uid: string) {
+    this.postsService.deleteUser(user_uid).subscribe(() => {
       this.getArray();
-    }, err => {
+    }, () => {
       this.getArray();
     });
   }
 
   createAndSortData() {
     // Sorts the data by last finished
-    this.users.sort((a, b) => b.name > a.name);
+    this.users.sort((a, b) => a.name.localeCompare(b.name));
 
     const filteredData = [];
     for (let i = 0; i < this.users.length; i++) {
@@ -188,7 +188,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
       }
     });
 
-    dialogRef.afterClosed().subscribe(success => {
+    dialogRef.afterClosed().subscribe(() => {
       this.getRoles();
     });
   }
@@ -197,7 +197,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     this.dialogRef.close();
   }
 
-  public openSnackBar(message: string, action: string = '') {
+  public openSnackBar(message: string, action = '') {
     this.snackBar.open(message, action, {
       duration: 2000,
     });

src/app/components/notifications-list/notifications-list.component.html

@@ -0,0 +1,30 @@
+<div class="card-radius mat-elevation-z2" *ngFor="let notification of notifications; let i = index;">
+    <mat-card class="notification-card card-radius">
+        <mat-card-header>
+            <mat-card-subtitle>
+                <div>
+                    <span class="notification-timestamp">{{notification.timestamp * 1000 | date:'short'}}</span>
+                </div>
+            </mat-card-subtitle>
+            <mat-card-title>
+                <ng-container *ngIf="NOTIFICATION_PREFIX[notification.type]">
+                    {{NOTIFICATION_PREFIX[notification.type]}}
+                </ng-container>
+            </mat-card-title>
+        </mat-card-header>
+        <mat-card-content>
+            <ng-container *ngIf="NOTIFICATION_SUFFIX_KEY[notification.type]">
+                <div style="word-break: break-word">
+                    {{notification['data'][NOTIFICATION_SUFFIX_KEY[notification.type]]}}
+                </div>
+            </ng-container>
+        </mat-card-content>
+        <mat-card-actions *ngIf="notification.actions?.length > 0">
+            <button matTooltip="Remove" i18n-matTooltip="Remove" (click)="emitDeleteNotification(notification.uid)" mat-icon-button><mat-icon>close</mat-icon></button>
+            <span *ngFor="let action of notification.actions">
+                <button [matTooltip]="NOTIFICATION_ACTION_TO_STRING[action]" (click)="emitNotificationAction(notification, action)" mat-icon-button><mat-icon>{{NOTIFICATION_ICON[action]}}</mat-icon></button>
+            </span>
+        </mat-card-actions>
+        <span *ngIf="!notification.read" class="dot"></span>
+    </mat-card>
+</div>

src/app/components/notifications-list/notifications-list.component.scss

@@ -0,0 +1,33 @@
+.notification-divider {
+    margin-bottom: 10px;
+    margin-top: 10px;
+}
+
+.notification-text {
+    margin-left: 10px;
+    margin-right: 10px;
+    display: inline-block;
+}
+
+.notification-timestamp {
+    font-size: 14px;
+}
+
+.notification-card {
+    margin-top: 5px;
+}
+
+.card-radius {
+    border-radius: 12px;
+}
+
+.dot {
+    height: 8px;
+    width: 8px;
+    background-color: red;
+    border-radius: 50%;
+    display: inline-block;
+    position: absolute;
+    right: 8px;
+    top: 8px;
+  }

src/app/components/notifications-list/notifications-list.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { NotificationsListComponent } from './notifications-list.component';
+
+describe('NotificationsListComponent', () => {
+  let component: NotificationsListComponent;
+  let fixture: ComponentFixture<NotificationsListComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ NotificationsListComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(NotificationsListComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/notifications-list/notifications-list.component.ts

@@ -0,0 +1,57 @@
+import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { Notification } from 'api-types';
+import { NotificationAction } from 'api-types/models/NotificationAction';
+import { NotificationType } from 'api-types/models/NotificationType';
+
+@Component({
+  selector: 'app-notifications-list',
+  templateUrl: './notifications-list.component.html',
+  styleUrls: ['./notifications-list.component.scss']
+})
+export class NotificationsListComponent {
+  @Input() notifications = null;
+  @Output() deleteNotification = new EventEmitter<string>();
+  @Output() notificationAction = new EventEmitter<{notification: Notification, action: NotificationAction}>();
+
+  NOTIFICATION_PREFIX: { [key in NotificationType]: string } = {
+    download_complete: $localize`Finished downloading`,
+    download_error: $localize`Download failed`,
+    task_finished: $localize`Task finished`
+  }
+
+  // Attaches string to the end of the notification text
+  NOTIFICATION_SUFFIX_KEY: { [key in NotificationType]: string } = {
+    download_complete: 'file_title',
+    download_error: 'download_url',
+    task_finished: 'task_title'
+  }
+
+  NOTIFICATION_ACTION_TO_STRING: { [key in NotificationAction]: string } = {
+    play: $localize`Play`,
+    retry_download: $localize`Retry download`,
+    view_download_error: $localize`View error`,
+    view_tasks: $localize`View task`
+  }
+
+  NOTIFICATION_COLOR: { [key in NotificationAction]: string } = {
+    play: 'primary',
+    retry_download: 'primary',
+    view_download_error: 'warn',
+    view_tasks: 'primary'
+  }
+
+  NOTIFICATION_ICON: { [key in NotificationAction]: string } = {
+    play: 'smart_display',
+    retry_download: 'restart_alt',
+    view_download_error: 'warning',
+    view_tasks: 'task'
+  }
+
+  emitNotificationAction(notification: Notification, action: NotificationAction): void {
+    this.notificationAction.emit({notification: notification, action: action});
+  }
+
+  emitDeleteNotification(uid: string): void {
+    this.deleteNotification.emit(uid);
+  }
+}

src/app/components/notifications/notifications.component.css

@@ -0,0 +1,10 @@
+.notification-title {
+    margin-bottom: 6px;
+    text-align: center
+}
+
+.notifications-list-parent {
+    max-height: 70vh;
+    overflow-y: auto;
+    padding: 0px 10px 10px 10px;
+}

src/app/components/notifications/notifications.component.html

@@ -0,0 +1,10 @@
+<div *ngIf="notifications !== null && notifications.length === 0" style="text-align: center; margin: 10px;" i18n="No notifications available">No notifications available</div>
+<div *ngIf="notifications?.length > 0">
+    <div class="notifications-list-parent">
+        <mat-chip-listbox [value]="selectedFilters" [multiple]="true" (change)="selectedFiltersChanged($event)">
+            <mat-chip-option *ngFor="let filter of notificationFilters | keyvalue: originalOrder" [value]="filter.key" [selected]="selectedFilters.includes(filter.key)" color="accent">{{filter.value.label}}</mat-chip-option>
+        </mat-chip-listbox>
+        <app-notifications-list (notificationAction)="notificationAction($event)" (deleteNotification)="deleteNotification($event)" [notifications]="filtered_notifications"></app-notifications-list>
+    </div>
+    <button style="margin: 10px 0px 2px 10px;" *ngIf="notifications?.length > 0" color="warn" (click)="deleteAllNotifications()" mat-stroked-button>Remove all</button>
+</div>

src/app/components/notifications/notifications.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { NotificationsComponent } from './notifications.component';
+
+describe('NotificationsComponent', () => {
+  let component: NotificationsComponent;
+  let fixture: ComponentFixture<NotificationsComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ NotificationsComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(NotificationsComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/notifications/notifications.component.ts

@@ -0,0 +1,124 @@
+import { Component, ElementRef, EventEmitter, OnInit, Output } from '@angular/core';
+import { Router } from '@angular/router';
+import { PostsService } from 'app/posts.services';
+import { Notification, NotificationType } from 'api-types';
+import { NotificationAction } from 'api-types/models/NotificationAction';
+import { MatChipListboxChange } from '@angular/material/chips';
+
+@Component({
+  selector: 'app-notifications',
+  templateUrl: './notifications.component.html',
+  styleUrls: ['./notifications.component.css']
+})
+export class NotificationsComponent implements OnInit {
+
+  notifications: Notification[] = null;
+  filtered_notifications: Notification[] = null;
+
+  @Output() notificationCount = new EventEmitter<number>();
+
+  notificationFilters: { [key in NotificationType]: {key: string, label: string} } = {
+    download_complete: {
+      key: 'download_complete',
+      label: $localize`Download completed`
+    },
+    download_error: {
+      key: 'download_error',
+      label: $localize`Download error`
+    },
+    task_finished: {
+      key: 'task_finished',
+      label: $localize`Task`
+    },
+  };
+
+  selectedFilters = [];
+
+  constructor(public postsService: PostsService, private router: Router, private elRef: ElementRef) { }
+
+  ngOnInit(): void {
+    // wait for init
+    if (this.postsService.initialized) {
+      this.getNotifications();
+    } else {
+      this.postsService.service_initialized.subscribe(init => {
+        if (init) {
+          this.getNotifications();
+        }
+      });
+    }
+  }
+
+  getNotifications(): void {
+    this.postsService.getNotifications().subscribe(res => {
+      this.notifications = res['notifications'];
+      this.notifications.sort((a, b) => b.timestamp - a.timestamp);
+      this.notificationCount.emit(this.notifications.filter(notification => !notification.read).length);
+
+      this.filterNotifications();
+    });
+  }
+
+  notificationAction(action_info: {notification: Notification, action: NotificationAction}): void {
+    switch (action_info['action']) {
+      case NotificationAction.PLAY:
+        this.router.navigate(['player', {uid: action_info['notification']['data']['file_uid']}]);
+        break;
+      case NotificationAction.VIEW_DOWNLOAD_ERROR:
+        this.router.navigate(['downloads']);
+        break;
+      case NotificationAction.RETRY_DOWNLOAD:
+        this.postsService.restartDownload(action_info['notification']['data']['download_uid']).subscribe(res => {
+          this.postsService.openSnackBar($localize`Download restarted!`);
+          this.deleteNotification(action_info['notification']['uid']);
+        });
+        break;
+      case NotificationAction.VIEW_TASKS:
+        this.router.navigate(['tasks']);
+        break;
+      default:
+        console.error(`Notification action ${action_info['action']} does not exist!`);
+        break;
+    }
+  }
+
+  deleteNotification(uid: string): void {
+    this.postsService.deleteNotification(uid).subscribe(res => {
+      this.notifications.filter(notification => notification['uid'] !== uid);
+      this.filterNotifications();
+      this.notificationCount.emit(this.notifications.length);
+      this.getNotifications();
+    });
+  }
+
+  deleteAllNotifications(): void {
+    this.postsService.deleteAllNotifications().subscribe(res => {
+      this.notifications = [];
+      this.filtered_notifications = [];
+      this.getNotifications();
+    });
+    this.notificationCount.emit(0);
+  }
+
+  setNotificationsToRead(): void {
+    const uids = this.notifications.map(notification => notification.uid);
+    this.postsService.setNotificationsToRead(uids).subscribe(res => {
+      this.getNotifications();
+    });
+    this.notificationCount.emit(0);
+  }
+
+  filterNotifications(): void {
+    this.filtered_notifications = this.notifications.filter(notification => this.selectedFilters.length === 0 || this.selectedFilters.includes(notification.type));
+  }
+
+  selectedFiltersChanged(event: MatChipListboxChange): void {
+    this.selectedFilters = event.value;
+    this.filterNotifications();
+  }
+
+  originalOrder = (): number => {
+    return 0;
+  }
+
+}

src/app/components/recent-videos/recent-videos.component.html

@@ -1,47 +1,47 @@
 <div class="container-fluid" style="max-width: 941px;">
     <div class="row">
+        <!-- Sorting -->
         <div class="col-12 order-2 col-sm-4 order-sm-1 d-flex justify-content-center">
-            <div>
-                <div style="display: inline-block;">
-                    <mat-form-field style="width: 132px;">
-                        <mat-select [(ngModel)]="this.filterProperty" (selectionChange)="filterOptionChanged($event.value)">
-                            <mat-option *ngFor="let filterOption of filterProperties | keyvalue" [value]="filterOption.value">
-                                {{filterOption['value']['label']}}
-                            </mat-option>
-                        </mat-select>
-                    </mat-form-field>
-                </div>
-                <div class="sort-dir-div">
-                    <button (click)="toggleModeChange()" mat-icon-button><mat-icon>{{descendingMode ? 'arrow_downward' : 'arrow_upward'}}</mat-icon></button>
-                </div>
-            </div>
+            <app-sort-property [(sortProperty)]="sortProperty" [(descendingMode)]="descendingMode" (sortOptionChanged)="sortOptionChanged($event)"></app-sort-property>
         </div>
+        <!-- Files title -->
         <div class="col-12 order-1 col-sm-4 order-sm-2 d-flex justify-content-center">
             <h4 *ngIf="!customHeader" class="my-videos-title" i18n="My files title">My files</h4>
             <h4 *ngIf="customHeader" class="my-videos-title">{{customHeader}}</h4>
         </div>
+        <!-- Search -->
         <div class="col-12 order-3 col-sm-4 order-sm-3 d-flex justify-content-center">
-            <mat-form-field [ngClass]="searchIsFocused ? 'search-bar-focused' : 'search-bar-unfocused'" class="search-bar" color="accent">
-                <input (focus)="searchIsFocused = true" (blur)="searchIsFocused = false" class="search-input" type="text" placeholder="Search" i18n-placeholder="Files search placeholder" [(ngModel)]="search_text" (ngModelChange)="onSearchInputChanged($event)" matInput>
+            <mat-form-field appearance="outline" [ngClass]="searchIsFocused ? 'search-bar-focused' : 'search-bar-unfocused'" class="search-bar" color="accent">
+                <mat-label i18n="Search">Search</mat-label>
+                <input (focus)="searchIsFocused = true" (blur)="searchIsFocused = false" class="search-input" type="text" [(ngModel)]="search_text" (ngModelChange)="onSearchInputChanged($event)" matInput>
                 <mat-icon matSuffix>search</mat-icon>
             </mat-form-field>
         </div>
     </div>
+    <!-- Filters -->
+    <div class="row justify-content-center">
+        <mat-chip-listbox class="filter-list" [value]="selectedFilters" [multiple]="true" (change)="selectedFiltersChanged($event)">
+            <mat-chip-option *ngFor="let filter of fileFilters | keyvalue: originalOrder" [value]="filter.key" [selected]="selectedFilters.includes(filter.key)" color="accent">{{filter.value.label}}</mat-chip-option>
+        </mat-chip-listbox>
+    </div>
 </div>
 <div>
+    <!-- Files -->
     <div *ngIf="!selectMode" class="container" style="margin-bottom: 16px">
         <div class="row justify-content-center">
+            <!-- Real cards -->
             <ng-container *ngIf="normal_files_received && paged_data">
                 <div style="display: flex; align-items: center;" *ngFor="let file of paged_data; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
-                    <app-unified-file-card [ngClass]="downloading_content[file.uid] ? 'blurred' : ''" [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [availablePlaylists]="playlists" (addFileToPlaylist)="addFileToPlaylist($event)" [loading]="false" (deleteFile)="deleteFile($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? this.postsService.token : ''"></app-unified-file-card>
+                    <app-unified-file-card [ngClass]="downloading_content[file.uid] ? 'blurred' : ''" [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" (toggleFavorite)="toggleFavorite($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [availablePlaylists]="playlists" (addFileToPlaylist)="addFileToPlaylist($event)" [loading]="false" (deleteFile)="deleteFile($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? this.postsService.token : ''"></app-unified-file-card>
                     <mat-spinner *ngIf="downloading_content[file.uid]" class="downloading-spinner" [diameter]="32"></mat-spinner>
                 </div>
                 <div *ngIf="paged_data.length === 0">
                     <ng-container i18n="No files found">No files found.</ng-container>
                 </div>
             </ng-container>
-            <ng-container *ngIf="!normal_files_received && loading_files && loading_files.length > 0">
-                <div *ngFor="let file of loading_files; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
+            <!-- Fake cards -->
+            <ng-container>
+                <div *ngFor="let file of loading_files; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[normal_files_received ? 'hide' : '', postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
                     <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" [loading]="true" [theme]="postsService.theme"></app-unified-file-card>
                 </div>
             </ng-container>
@@ -49,6 +49,7 @@
     </div>
 
     <div *ngIf="selectMode">
+        <!-- If selected files e.g. for creating a playlist -->
         <mat-tab-group [(selectedIndex)]="selectedIndex">
             <mat-tab label="Order" i18n-label="Order">
                 <div *ngIf="selected_data.length">
@@ -73,8 +74,8 @@
                     <mat-list-option [selected]="selected_data.includes(file.uid)" *ngFor="let file of paged_data" [value]="file">
                         <div class="container">
                             <div class="row justify-content-center">
-                                <div class="col-10">
-                                    <mat-icon class="audio-video-icon">{{(file.type === 'audio' || file.isAudio) ? 'audiotrack' : 'movie'}}</mat-icon>
+                                <div class="col-10 select-file-title">
+                                    <mat-icon class="audio-video-icon">{{file.isAudio ? 'audiotrack' : 'movie'}}</mat-icon>
                                     {{file.title}}
                                 </div>
                                 <div class="col-2">{{file.registered | date:'shortDate'}}</div>
@@ -87,7 +88,7 @@
                 <ng-container *ngIf="!normal_files_received && loading_files && loading_files.length > 0">
                     <mat-selection-list *ngIf="!normal_files_received">
                         <mat-list-option *ngFor="let file of paged_data">
-                            <content-loader class="list-ghosts" [primaryColor]="postsService.theme.ghost_primary" [secondaryColor]="postsService.theme.ghost_secondary" width="250" height="8"><svg:rect x="0" y="0" rx="3" ry="3" width="250" height="8" /></content-loader>
+                            <content-loader class="list-ghosts" [primaryColor]="postsService.theme.ghost_primary" [secondaryColor]="postsService.theme.ghost_secondary" [width]="250" [height]="8"><svg:rect x="0" y="0" rx="3" ry="3" width="250" height="8" /></content-loader>
                         </mat-list-option>
                     </mat-selection-list>
                 </ng-container>
@@ -96,16 +97,6 @@
     </div>
 
     <div style="position: relative;" *ngIf="usePaginator && selectedIndex > 0">
-        <div style="position: absolute; margin-left: 8px; margin-top: 5px; scale: 0.8">
-            <mat-form-field>
-                <mat-label><ng-container i18n="File type">File type</ng-container></mat-label>
-                <mat-select color="accent" [(ngModel)]="fileTypeFilter" (selectionChange)="fileTypeFilterChanged($event.value)">
-                    <mat-option value="both"><ng-container i18n="Both">Both</ng-container></mat-option>
-                    <mat-option value="video_only"><ng-container i18n="Video only">Video only</ng-container></mat-option>
-                    <mat-option value="audio_only"><ng-container i18n="Audio only">Audio only</ng-container></mat-option>
-                </mat-select>
-            </mat-form-field>
-        </div>
         <mat-paginator class="paginator" #paginator (page)="pageChangeEvent($event)" [length]="file_count"
                 [pageSize]="pageSize"
                 [pageSizeOptions]="[5, 10, 25, 100, this.paged_data && this.paged_data.length > 100 ? this.paged_data.length : 250]">

src/app/components/recent-videos/recent-videos.component.scss

@@ -17,7 +17,7 @@
 }
  
 .search-bar-unfocused {
-    width: 132px;
+    width: 165px;
 }
 
 .search-input {
@@ -41,12 +41,6 @@
     display: inline-block;
 }
 
-.sort-dir-div {
-    display: inline-block;
-    position: absolute;
-    top: 10px;
-}
-
 .paginator {
     margin-top: 5px;
 }
@@ -100,7 +94,7 @@
 .remove-item-button {
     right: 10px;
     position: absolute;
-    top: 4px;
+    top: 0px;
 }
 
 .playlist-item-text {
@@ -118,4 +112,18 @@
 .downloading-spinner {
     align-self: center;
     position: absolute;
+}
+
+.filter-list {
+    margin-bottom: 10px;
+}
+
+.hide {
+    display: none !important;
+}
+
+.select-file-title {
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    overflow: hidden;
 }

src/app/components/recent-videos/recent-videos.component.ts

@@ -1,11 +1,13 @@
 import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core';
 import { PostsService } from 'app/posts.services';
 import { Router } from '@angular/router';
-import { DatabaseFile, FileType, FileTypeFilter } from '../../../api-types';
+import { DatabaseFile, FileType, FileTypeFilter, Sort } from '../../../api-types';
 import { MatPaginator } from '@angular/material/paginator';
 import { Subject } from 'rxjs';
 import { distinctUntilChanged } from 'rxjs/operators';
 import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop';
+import { MatChipListboxChange } from '@angular/material/chips';
+import { MatSelectionListChange } from '@angular/material/list';
 
 @Component({
   selector: 'app-recent-videos',
@@ -46,35 +48,27 @@ export class RecentVideosComponent implements OnInit {
   search_text = '';
   searchIsFocused = false;
   descendingMode = true;
-  filterProperties = {
-    'registered': {
-      'key': 'registered',
-      'label': 'Download Date',
-      'property': 'registered'
+
+  fileFilters = {
+    video_only: {
+      key: 'video_only',
+      label: $localize`Video only`,
+      incompatible: ['audio_only']
     },
-    'upload_date': {
-      'key': 'upload_date',
-      'label': 'Upload Date',
-      'property': 'upload_date'
+    audio_only: {
+      key: 'audio_only',
+      label: $localize`Audio only`,
+      incompatible: ['video_only']
     },
-    'name': {
-      'key': 'name',
-      'label': 'Name',
-      'property': 'title'
+    favorited: {
+      key: 'favorited',
+      label: $localize`Favorited`
     },
-    'file_size': {
-      'key': 'file_size',
-      'label': 'File Size',
-      'property': 'size'
-    },
-    'duration': {
-      'key': 'duration',
-      'label': 'Duration',
-      'property': 'duration'
-    }
   };
-  filterProperty = this.filterProperties['upload_date'];
-  fileTypeFilter = 'both';
+
+  selectedFilters = [];
+
+  sortProperty = 'registered';
   
   playlists = null;
 
@@ -82,21 +76,24 @@ export class RecentVideosComponent implements OnInit {
 
   constructor(public postsService: PostsService, private router: Router) {
     // get cached file count
-    if (localStorage.getItem('cached_file_count')) {
-      this.cached_file_count = +localStorage.getItem('cached_file_count') <= 10 ? +localStorage.getItem('cached_file_count') : 10;
+    const sub_id_appendix = this.sub_id ? `_${this.sub_id}` : ''
+    if (localStorage.getItem(`cached_file_count${sub_id_appendix}`)) {
+      this.cached_file_count = +localStorage.getItem(`cached_file_count${sub_id_appendix}`) <= 10 ? +localStorage.getItem(`cached_file_count${sub_id_appendix}`) : 10;
       this.loading_files = Array(this.cached_file_count).fill(0);
     }
 
     // set filter property to cached value
-    const cached_filter_property = localStorage.getItem('filter_property');
-    if (cached_filter_property && this.filterProperties[cached_filter_property]) {
-      this.filterProperty = this.filterProperties[cached_filter_property];
+    const cached_sort_property = localStorage.getItem('sort_property');
+    if (cached_sort_property) {
+      this.sortProperty = cached_sort_property;
     }
 
     // set file type filter to cached value
-    const cached_file_type_filter = localStorage.getItem('file_type_filter');
-    if (this.usePaginator && cached_file_type_filter) {
-      this.fileTypeFilter = cached_file_type_filter;
+    const cached_file_filter = localStorage.getItem('file_filter');
+    if (this.usePaginator && cached_file_filter) {
+      this.selectedFilters = JSON.parse(cached_file_filter)
+    } else {
+      this.selectedFilters = [];
     }
 
     const sort_order = localStorage.getItem('recent_videos_sort_order');
@@ -107,6 +104,12 @@ export class RecentVideosComponent implements OnInit {
   }
 
   ngOnInit(): void {
+    if (this.sub_id) {
+      // subscriptions can't download both audio and video (for now), so don't let users filter for these
+      delete this.fileFilters['audio_only'];
+      delete this.fileFilters['video_only'];
+    }
+
     if (this.postsService.initialized) {
       this.getAllFiles();
       this.getAllPlaylists();
@@ -161,30 +164,59 @@ export class RecentVideosComponent implements OnInit {
     this.searchChangedSubject.next(newvalue);
   }
 
-  filterOptionChanged(value: string): void {
-    localStorage.setItem('filter_property', value['key']);
+  sortOptionChanged(value: Sort): void {
+    localStorage.setItem('sort_property', value['by']);
+    localStorage.setItem('recent_videos_sort_order', value['order'] === -1 ? 'descending' : 'ascending');
+    this.descendingMode = value['order'] === -1;
+    this.sortProperty = value['by'];
+    
     this.getAllFiles();
   }
 
-  fileTypeFilterChanged(value: string): void {
-    localStorage.setItem('file_type_filter', value);
-    this.getAllFiles();
+  filterChanged(value: string): void {
+    localStorage.setItem('file_filter', value);
+    // wait a bit for the animation to finish
+    setTimeout(() => this.getAllFiles(), 150);
   }
 
-  toggleModeChange(): void {
-    this.descendingMode = !this.descendingMode;
-    localStorage.setItem('recent_videos_sort_order', this.descendingMode ? 'descending' : 'ascending');
-    this.getAllFiles();
+  selectedFiltersChanged(event: MatChipListboxChange): void {
+    // in some cases this function will fire even if the selected filters haven't changed
+    if (event.value.length === this.selectedFilters.length) return;
+    if (event.value.length > this.selectedFilters.length) {
+      const filter_key = event.value.filter(possible_new_key => !this.selectedFilters.includes(possible_new_key))[0];
+      this.selectedFilters = this.selectedFilters.filter(existing_filter => !this.fileFilters[existing_filter].incompatible || !this.fileFilters[existing_filter].incompatible.includes(filter_key));
+      this.selectedFilters.push(filter_key);
+    } else {
+      this.selectedFilters = event.value;
+    }
+    this.filterChanged(JSON.stringify(this.selectedFilters));
   }
 
+  getFileTypeFilter(): string {
+    if (this.selectedFilters.includes('audio_only')) {
+      return 'audio_only';
+    } else if (this.selectedFilters.includes('video_only')) {
+      return 'video_only';
+    } else {
+      return 'both';
+    }
+  }
+
+  getFavoriteFilter(): boolean {
+    return this.selectedFilters.includes('favorited');
+  }
+
+
   // get files
 
   getAllFiles(cache_mode = false): void {
     this.normal_files_received = cache_mode;
     const current_file_index = (this.paginator?.pageIndex ? this.paginator.pageIndex : 0)*this.pageSize;
-    const sort = {by: this.filterProperty['property'], order: this.descendingMode ? -1 : 1};
+    const sort = {by: this.sortProperty, order: this.descendingMode ? -1 : 1};
     const range = [current_file_index, current_file_index + this.pageSize];
-    this.postsService.getAllFiles(sort, range, this.search_mode ? this.search_text : null, this.fileTypeFilter as FileTypeFilter, this.sub_id).subscribe(res => {
+    const fileTypeFilter = this.getFileTypeFilter();
+    const favoriteFilter = this.getFavoriteFilter();
+    this.postsService.getAllFiles(sort, this.usePaginator ? range : null, this.search_mode ? this.search_text : null, fileTypeFilter as FileTypeFilter, favoriteFilter, this.sub_id).subscribe(res => {
       this.file_count = res['file_count'];
       this.paged_data = res['files'];
       for (let i = 0; i < this.paged_data.length; i++) {
@@ -286,16 +318,14 @@ export class RecentVideosComponent implements OnInit {
   }
 
   deleteAndRedownload(file: DatabaseFile): void {
-    const sub = this.postsService.getSubscriptionByID(file.sub_id);
-    this.postsService.deleteSubscriptionFile(sub, file.id, false, file.uid).subscribe(() => {
+    this.postsService.deleteSubscriptionFile(file.uid, false).subscribe(() => {
       this.postsService.openSnackBar($localize`Successfully deleted file: ` + file.id);
       this.removeFileCard(file);
     });
   }
 
   deleteForever(file: DatabaseFile): void {
-    const sub = this.postsService.getSubscriptionByID(file.sub_id);
-    this.postsService.deleteSubscriptionFile(sub, file.id, true, file.uid).subscribe(() => {
+    this.postsService.deleteSubscriptionFile(file.uid, true).subscribe(() => {
       this.postsService.openSnackBar($localize`Successfully deleted file: ` + file.id);
       this.removeFileCard(file);
     });
@@ -348,8 +378,9 @@ export class RecentVideosComponent implements OnInit {
     this.getAllFiles();
   }
 
-  fileSelectionChanged(event: { option: { _selected: boolean; value: DatabaseFile; } }): void {
-    const adding = event.option._selected;
+  fileSelectionChanged(event: MatSelectionListChange): void {
+    // TODO: make sure below line is possible (_selected is private)
+    const adding = event.option['_selected'];
     const value = event.option.value;
     if (adding) {
       this.selected_data.push(value.uid);
@@ -385,4 +416,13 @@ export class RecentVideosComponent implements OnInit {
     this.selected_data_objs.splice(index, 1);
     this.fileSelectionEmitter.emit({new_selection: this.selected_data, thumbnailURL: this.selected_data_objs[0].thumbnailURL});
   }
+
+  originalOrder = (): number => {
+    return 0;
+  }
+
+  toggleFavorite(file_obj): void {
+    file_obj.favorite = !file_obj.favorite;
+    this.postsService.updateFile(file_obj.uid, {favorite: file_obj.favorite}).subscribe(res => {});
+  }
 }

src/app/components/sort-property/sort-property.component.html

@@ -0,0 +1,14 @@
+<div>
+    <div style="display: inline-block;">
+        <mat-form-field appearance="outline" style="width: 165px;">
+            <mat-select [(ngModel)]="this.sortProperty" (selectionChange)="emitSortOptionChanged()">
+                <mat-option *ngFor="let sortOption of sortProperties | keyvalue" [value]="sortOption.key">
+                    {{sortOption['value']['label']}}
+                </mat-option>
+            </mat-select>
+        </mat-form-field>
+    </div>
+    <div class="sort-dir-div">
+        <button (click)="toggleModeChange()" mat-icon-button><mat-icon>{{descendingMode ? 'arrow_downward' : 'arrow_upward'}}</mat-icon></button>
+    </div>
+</div>