Improved tests for multi-user mode

Pre 4.3.1 bug fixes

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
       - name: setup node
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
           cache: 'npm'
       - name: install dependencies
         run: |

.github/workflows/docker-release.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Set image tag
         id: tags
         run: |
-          if [ ${{ github.event.action }} == "workflow_dispatch" ]; then
+          if [ "${{ github.event.inputs.tags }}" != "" ]; then
             echo "::set-output name=tags::${{ github.event.inputs.tags }}"
           elif [ ${{ github.event.action }} == "release" ]; then
             echo "::set-output name=tags::${{ github.event.release.tag_name }}"
@@ -53,8 +53,8 @@ jobs:
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
             ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
           tags: |
-            raw=${{ steps.tags.outputs.tags }}
-            raw=latest
+            type=raw,value=${{ steps.tags.outputs.tags }}
+            type=raw,value=latest
 
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1

.github/workflows/docker.yml

@@ -13,6 +13,9 @@ on:
       - '**.pem'
       - '.dockerignore'
       - '.gitignore'
+  schedule:
+    - cron:  '34 4 * * 2'
+  workflow_dispatch:
 
 jobs:
   build-and-push:

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+    "recommendations": [
+        "angular.ng-template",
+        "dbaeumer.vscode-eslint",
+        "waderyan.gitblame",
+        "42crunch.vscode-openapi",
+        "redhat.vscode-yaml",
+        "christian-kohler.npm-intellisense",
+        "hbenl.vscode-mocha-test-adapter"
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "mochaExplorer.files": "backend/test/**/*.js",
+    "mochaExplorer.cwd": "backend",
+    "mochaExplorer.globImplementation": "vscode",
+    "mochaExplorer.env": {
+        "YTDL_MODE": "debug"
+    }
+}

.vscode/tasks.json

@@ -1,25 +1,60 @@
 {
 	"version": "2.0.0",
+	"windows": {
+        "options": {
+            "shell": {
+                "executable": "cmd.exe",
+                "args": [
+                    "/d", "/c"
+                ]
+            }
+        }
+    },
 	"tasks": [
 		{
 			"type": "npm",
 			"script": "start",
 			"problemMatcher": [],
 			"label": "Dev: start frontend",
-			"detail": "ng serve"
+			"detail": "ng serve",
+			"presentation": {
+				"echo": true,
+				"reveal": "always",
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
+			}
 		},
 		{
 			"label": "Dev: start backend",
 			"type": "shell",
-			"command": "set YTDL_MODE=debug && node app.js",
+			"command": "node app.js",
 			"options": {
-				"cwd": "./backend"
+				"cwd": "./backend",
+				"env": {
+					"YTDL_MODE": "debug"
+				}
 			},
 			"presentation": {
+				"echo": true,
 				"reveal": "always",
-				"panel": "new"
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
 			},
-			"problemMatcher": []
+			"problemMatcher": [],
+			"dependsOn": ["Dev: post-build"]
+		},
+		{
+			"label": "Dev: post-build",
+			"type": "shell",
+			"command": "node src/postbuild.mjs"
+		},
+		{
+			"label": "Dev: run all",
+			"dependsOn": ["Dev: start backend", "Dev: start frontend"]
 		}
 	]
 }

Dockerfile

@@ -2,25 +2,28 @@
 FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY ffmpeg-fetch.sh .
+COPY docker-utils/ffmpeg-fetch.sh .
+RUN chmod +x ffmpeg-fetch.sh
 RUN sh ./ffmpeg-fetch.sh
 
 
-# Create our Ubuntu 22.04 with node 16
+# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
 # Go to 20.04
 FROM ubuntu:20.04 AS base
-ENV DEBIAN_FRONTEND=noninteractive
+ARG DEBIAN_FRONTEND=noninteractive
 ENV UID=1000
 ENV GID=1000
 ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
 ENV PM2_HOME=/app/pm2
+ENV ALLOW_CONFIG_MUTATIONS=true
 RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
     apt update && \
-    apt install -y --no-install-recommends curl ca-certificates && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
     curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
     apt install -y --no-install-recommends nodejs && \
-    npm -g install npm && \
+    npm -g install npm n && \
+    n 16.14.2 && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -44,23 +47,35 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
 RUN npm install -g pm2 && \
     apt update && \
-    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 atomicparsley && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
+RUN pip install pycryptodomex
 WORKDIR /app
 # User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
 COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
+RUN chown $UID:$GID .
+RUN chmod +x /app/fix-scripts/*.sh
 # Add some persistence data
 #VOLUME ["/app/appdata"]
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime","--raw","pm2.config.js" ]
+CMD [ "npm","start" ]

Dockerfile.heroku

@@ -1,2 +1,2 @@
-FROM tzahi12345/youtubedl-material:nightly
-CMD [ "pm2-runtime", "pm2.config.js" ]
+FROM tzahi12345/youtubedl-material:latest
+CMD [ "npm", "start" ]

Public API v1.yaml

@@ -97,6 +97,11 @@ paths:
       summary: Get all files
       description: Gets all files and playlists stored in the db
       operationId: get-getAllFiles
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GetAllFilesRequest'
       responses:
         '200':
           description: OK
@@ -106,6 +111,37 @@ paths:
                 $ref: '#/components/schemas/GetAllFilesResponse'
       security:
         - Auth query parameter: []
+  /api/rss:
+    get:
+      tags:
+        - files
+      summary: Generates an RSS feed
+      description: Generates an RSS feed for downloaded files
+      operationId: get-rss
+      parameters:
+        - in: query
+          name: params
+          schema:
+            allOf:
+            - $ref: '#/components/schemas/GetAllFilesRequest'
+            - type: object
+              properties:
+                uuid:
+                  type: string
+                  description: user uid
+                  default: null
+          style: form
+          explode: true
+      responses:
+        '200':
+          description: OK
+          content:
+            text/plain:
+              schema:
+                type: string
+                description: RSS feed
+      security:
+        - Auth query parameter: []
   /api/getFile:
     post:
       tags:
@@ -542,6 +578,69 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
+  /api/deleteArchiveItems:
+    post:
+      tags:
+        - archive
+      summary: Delete item from archive
+      description: 'Deletes an item from the archive'
+      operationId: post-api-deleteArchiveItems
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DeleteArchiveItemsRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/importArchive:
+    post:
+      tags:
+        - archive
+      summary: Imports archive
+      description: 'Imports an existing archive.txt file'
+      operationId: post-api-importArchive
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ImportArchiveRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/uploadCookies:
+    post:
+      tags:
+        - downloader
+      summary: Upload cookies
+      description: 'Uploads cookies file to be used during downloading'
+      operationId: post-api-uploadCookies
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadCookiesRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -806,7 +905,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/SuccessObject'
+                $ref: '#/components/schemas/RestartDownloadResponse'
       requestBody:
         content:
           application/json:
@@ -1584,6 +1683,10 @@ components:
           type: string
           description: Height of the video, if known
           example: '1080'
+        maxHeight:
+          type: string
+          description: Max height that should be used, useful for playlists. selectedHeight will override this.
+          example: '1080'
         maxBitrate:
           type: string
           description: Specify ffmpeg/avconv audio quality
@@ -1592,6 +1695,9 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
+        ignoreArchive:
+          type: boolean
+          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -1616,6 +1722,13 @@ components:
       properties:
         download:
           $ref: '#/components/schemas/Download'
+    RestartDownloadResponse:
+      allOf:
+        - $ref: '#/components/schemas/SuccessObject'
+        - type: object
+          properties:
+            new_download_uid:
+              type: string
     GetAllDownloadsRequest:
       type: object
       properties:
@@ -1668,6 +1781,16 @@ components:
       required:
         - task_key
         - new_data
+    UpdateTaskOptionsRequest:
+      type: object
+      properties:
+        task_key:
+          type: string
+        new_options:
+          type: object
+      required:
+        - task_key
+        - new_options
     GetTaskResponse:
       type: object
       properties:
@@ -1724,6 +1847,51 @@ components:
           description: All video playlists
           items:
             $ref: '#/components/schemas/Playlist'
+    GetAllFilesRequest:
+      type: object
+      properties:
+        sort:
+          $ref: '#/components/schemas/Sort'
+        range:
+          type: array
+          items:
+            type: number
+            description: Two elements allowed, start index and end index
+          minItems: 2
+          maxItems: 2
+          default: null
+        text_search:
+          type: string
+          description: Filter files by title
+          default: null
+        file_type_filter:
+          $ref: '#/components/schemas/FileTypeFilter'
+        favorite_filter:
+          type: boolean
+          description: If set to true, only gets favorites
+          default: false
+        sub_id:
+          type: string
+          description: Include if you want to filter by subscription
+          default: null
+    Sort:
+      type: object
+      properties:
+        by:
+          type: string
+          description: Property to sort by
+          default: registered
+        order:
+          type: number
+          description: 1 for ascending, -1 for descending
+          default: -1
+    FileTypeFilter:
+      type: string
+      enum:
+        - audio_only
+        - video_only
+        - both
+      default: both
     GetAllFilesResponse:
       required:
         - files
@@ -1786,7 +1954,6 @@ components:
       required:
         - name
         - url
-        - streamingOnly
       type: object
       properties:
         name:
@@ -1842,16 +2009,11 @@ components:
           description: Number of files removed
     DeleteSubscriptionFileRequest:
       required:
-        - file
-        - sub
+        - file_uid
       type: object
       properties:
-        file:
-          type: string
         file_uid:
           type: string
-        sub:
-          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteForever:
           type: boolean
           description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -1899,7 +2061,6 @@ components:
         - uids
         - playlistName
         - thumbnailURL
-        - type
       type: object
       properties:
         playlistName:
@@ -1908,8 +2069,6 @@ components:
           type: array
           items:
             type: string
-        type:
-          $ref: '#/components/schemas/FileType'
         thumbnailURL:
           type: string
     CreatePlaylistResponse:
@@ -1939,15 +2098,17 @@ components:
       required:
         - playlist
         - success
-        - type
       type: object
       properties:
         playlist:
           $ref: '#/components/schemas/Playlist'
-        type:
-          $ref: '#/components/schemas/FileType'
         success:
           type: boolean
+        file_objs:
+          type: array
+          description: File objects for every uid in the playlist's uids property, in the same order
+          items:
+            $ref: '#/components/schemas/DatabaseFile'
     GetPlaylistsRequest:
       type: object
       properties:
@@ -1972,13 +2133,10 @@ components:
     DeletePlaylistRequest:
       required:
         - playlist_id
-        - type
       type: object
       properties:
         playlist_id:
           type: string
-        type:
-          $ref: '#/components/schemas/FileType'
     DownloadFileRequest:
       type: object
       properties:
@@ -1995,17 +2153,83 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      required:
-        - sub
       type: object
       properties:
-        sub:
-          required:
-            - archive_dir
-          type: object
-          properties:
-            archive_dir:
-              type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    Archive:
+      required:
+        - extractor
+        - id
+        - type
+        - title
+        - timestamp
+        - uid
+      type: object
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        title:
+          type: string
+        user_uid:
+          type: string
+        sub_id:
+          type: string
+        timestamp:
+          type: number
+        uid:
+          type: string
+    DeleteArchiveItemsRequest:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    ImportArchiveRequest:
+      type: object
+      required:
+        - archive
+        - type
+      properties:
+        archive:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesRequest:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesResponse:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    UploadCookiesRequest:
+      type: object
+      required:
+        - cookies
+      properties:
+        cookies:
+          type: string
+          format: binary
     UpdaterStatus:
       required:
         - details
@@ -2026,8 +2250,6 @@ components:
         tag:
           type: string
     DBInfoResponse:
-      required:
-        - db_info
       type: object
       properties:
         using_local_db:
@@ -2049,6 +2271,8 @@ components:
               $ref: '#/components/schemas/TableInfo'
             download_queue:
               $ref: '#/components/schemas/TableInfo'
+            archives:
+              $ref: '#/components/schemas/TableInfo'
     TransferDBResponse:
       required:
         - success
@@ -2348,6 +2572,7 @@ components:
         - upload_date
         - uploader
         - url
+        - favorite
       type: object
       properties:
         id:
@@ -2377,6 +2602,8 @@ components:
           type: string
         uid:
           type: string
+        user_uid:
+          type: string
         sharingEnabled:
           type: boolean
         category:
@@ -2385,6 +2612,18 @@ components:
           type: number
         local_view_count:
           type: number
+        sub_id:
+          type: string
+        registered:
+          type: number
+        height:
+          type: number
+          description: In pixels, only for videos
+        abr:
+          type: number
+          description: In Kbps
+        favorite:
+          type: boolean
     Playlist:
       required:
         - uids
@@ -2416,6 +2655,8 @@ components:
           type: string
         auto:
           type: boolean
+        sharingEnabled:
+          type: boolean
     Download:
       required:
         - url
@@ -2460,12 +2701,18 @@ components:
           type: string
           description: Error text, set if download fails.
           nullable: true
+        error_type:
+          type: string
+          description: Error type, may or may not be set in case of an error
+          nullable: true
         user_uid:
           type: string
         sub_id:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key
@@ -2480,6 +2727,8 @@ components:
       properties:
         key:
           type: string
+        title:
+          type: string
         last_ran:
           type: number
         last_confirmed:
@@ -2494,6 +2743,8 @@ components:
           type: string
         schedule:
           type: object
+        options:
+          type: object
     Schedule:
       required:
         - type
@@ -2518,6 +2769,8 @@ components:
               type: number
             timestamp:
               type: number
+            tz:
+              type: string
     DBBackup:
       required:
         - name
@@ -2560,7 +2813,6 @@ components:
         - url
         - type
         - user_uid
-        - streamingOnly
         - isPlaylist
         - videos
       type: object
@@ -2576,8 +2828,6 @@ components:
         user_uid:
           type: string
           nullable: true
-        streamingOnly:
-          type: boolean
         isPlaylist:
           type: boolean
         archive:
@@ -2709,6 +2959,44 @@ components:
           type: string
         date:
           type: string
+    Notification:
+      required:
+        - uid
+        - type
+        - text
+        - read
+        - timestamp
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/NotificationType'
+        uid:
+          type: string
+        user_uid:
+          type: string
+        action:
+          type: array
+          items:
+            $ref: '#/components/schemas/NotificationAction'
+        read:
+          type: boolean
+        data:
+          type: object
+        timestamp:
+          type: number
+    NotificationAction:
+      type: string
+      enum:
+      - play
+      - retry_download
+      - view_download_error
+      - view_tasks
+    NotificationType:
+      type: string
+      enum:
+      - download_complete
+      - download_error
+      - task_finished
     BaseChangePermissionsRequest:
       required:
         - permission
@@ -2840,6 +3128,29 @@ components:
                   type: array
                   items:
                     $ref: '#/components/schemas/UserPermission'
+    DeleteNotificationRequest:
+      required:
+        - uid
+      type: object
+      properties:
+        uid:
+          type: string
+    SetNotificationsToReadRequest:
+      required:
+        - uids
+      type: object
+      properties:
+        uids:
+          type: array
+          items:
+            type: string
+    GetNotificationsResponse:
+      type: object
+      properties:
+        notifications:
+          type: array
+          items:
+            $ref: '#/components/schemas/Notification'
   securitySchemes:
     Auth query parameter:
       name: apiKey

README.md

@@ -6,25 +6,15 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
 <hr>
 
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
-Check out the prerequisites, and go to the installation section. Easy as pie!
+Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
@@ -58,9 +48,12 @@ sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 Optional dependencies:
 
 * AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
+If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
+
 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.
@@ -91,7 +84,7 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ### Host-specific instructions
 
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
 
 ### Setup
 
@@ -102,8 +95,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for
 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
 
-NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
-
 ### Custom UID/GID
 
 By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
@@ -114,6 +105,12 @@ environment:
     GID: YOUR_GID
 ```
 
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
 ## API
 
 [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)

SECURITY.md

@@ -2,16 +2,16 @@
 
 ## Supported Versions
 
-Currently all work on this project goes into the nightly builds.
-4.2's RELEASE build is now quite old and should be considered legacy.
-We urge users to use the nightly releases, because the project
-constantly sees fixes.
+If you would like to see the latest updates, use the `nightly` tag on Docker.
 
-| Version       | Supported          |
-| ------------- | ------------------ |
-| 4.2 Nightlies | :white_check_mark: |
-| 4.2 Release   | :x:                |
-| < 4.2         | :x:                |
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
+
+| Version              | Supported          |
+| -------------------- | ------------------ |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
 
 ## Reporting a Vulnerability
 

angular.json

@@ -182,7 +182,6 @@
       }
     }
   },
-  "defaultProject": "youtube-dl-material",
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",
@@ -191,5 +190,8 @@
     "@schematics/angular:directive": {
       "prefix": "app"
     }
+  },
+  "cli": {
+    "analytics": false
   }
 }

backend/app.js

@@ -18,6 +18,7 @@ const URL = require('url').URL;
 const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
+const Feed = require('feed').Feed;
 
 // needed if bin/details somehow gets deleted
 if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -32,6 +33,7 @@ const subscriptions_api = require('./subscriptions');
 const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 var app = express();
 
@@ -68,7 +70,9 @@ db.defaults(
         configWriteFlag: false,
         downloads: {},
         subscriptions: [],
-        files_to_db_migration_complete: false
+        files_to_db_migration_complete: false,
+        tasks_manager_role_migration_complete: false,
+        archives_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -101,7 +105,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -148,22 +151,18 @@ if (fs.existsSync('version.json')) {
 
 // don't overwrite config if it already happened.. NOT
 // let alreadyWritten = db.get('configWriteFlag').value();
-let writeConfigMode = process.env.write_ytdl_config;
 
 // checks if config exists, if not, a config is auto generated
 config_api.configExistsCheck();
 
-if (writeConfigMode) {
-    setAndLoadConfig();
-} else {
-    loadConfig();
-}
+setAndLoadConfig();
 
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
 // use passport
 app.use(auth_api.passport.initialize());
+app.use(auth_api.passport.session());
 
 // actual functions
 
@@ -188,13 +187,31 @@ async function checkMigrations() {
     if (!new_db_system_migration_complete) {
         logger.info('Beginning migration: 4.2->4.3+')
         let success = await db_api.importJSONToDB(db.value(), users_db.value());
-
+        await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first
         // sets migration to complete
         db.set('new_db_system_migration_complete', true).write();
         if (success) { logger.info('4.2->4.3+ migration complete!'); }
         else { logger.error('Migration failed: 4.2->4.3+'); }
     }
 
+    const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value();
+    if (!tasks_manager_role_migration_complete) {
+        logger.info('Checking if tasks manager role permissions exist for admin user...');
+        const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes');
+        if (success) logger.info('Task manager permissions check complete!');
+        else logger.error('Failed to auto add tasks manager permissions to admin role!');
+        db.set('tasks_manager_role_migration_complete', true).write();
+    }
+
+    const archives_migration_complete = db.get('archives_migration_complete').value();
+    if (!archives_migration_complete) {
+        logger.info('Checking if archives have been migrated...');
+        const imported_archives = await archive_api.importArchives();
+        if (imported_archives) logger.info('Archives migration complete!');
+        else logger.error('Failed to migrate archives!');
+        db.set('archives_migration_complete', true).write();
+    }
+
     return true;
 }
 
@@ -484,8 +501,9 @@ async function setAndLoadConfig() {
 }
 
 async function setConfigFromEnv() {
-    let config_items = getEnvConfigItems();
-    let success = config_api.setConfigItems(config_items);
+    const config_items = getEnvConfigItems();
+    if (!config_items || config_items.length === 0) return true;
+    const success = config_api.setConfigItems(config_items);
     if (success) {
         logger.info('Config items set using ENV variables.');
         await utils.wait(100);
@@ -500,13 +518,11 @@ async function loadConfig() {
     loadConfigValues();
 
     // connect to DB
-    await db_api.connectToDB();
+    if (!config_api.getConfigItem('ytdl_use_local_db'))
+        await db_api.connectToDB();
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
-    // creates archive path if missing
-    await fs.ensureDir(archivePath);
-
     // check migrations
     await checkMigrations();
 
@@ -552,14 +568,7 @@ function loadConfigValues() {
     url_domain = new URL(url);
 
     let logger_level = config_api.getConfigItem('ytdl_logger_level');
-    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
-    if (!possible_levels.includes(logger_level)) {
-        logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
-        logger_level = 'info';
-    }
-    logger.level = logger_level;
-    winston.loggers.get('console').level = logger_level;
-    logger.transports[2].level = logger_level;
+    utils.updateLoggerLevel(logger_level);
 }
 
 function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -575,7 +584,11 @@ async function watchSubscriptions() {
 
     if (!subscriptions) return;
 
-    const valid_subscriptions = subscriptions.filter(sub => !sub.paused);
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
 
     let subscriptions_amount = valid_subscriptions.length;
     let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
@@ -690,7 +703,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -763,7 +776,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
 
 app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
     const db_info = await db_api.getDBStats();
-    res.send({db_info: db_info});
+    res.send(db_info);
 });
 
 app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -803,11 +816,13 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings
+        cropFileSettings: req.body.cropFileSettings,
+        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -833,6 +848,7 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
@@ -911,31 +927,15 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
-    let files = null;
-    let playlists = null;
-    let sort = req.body.sort;
-    let range = req.body.range;
-    let text_search = req.body.text_search;
-    let file_type_filter = req.body.file_type_filter;
+    const sort = req.body.sort;
+    const range = req.body.range;
+    const text_search = req.body.text_search;
+    const file_type_filter = req.body.file_type_filter;
+    const favorite_filter = req.body.favorite_filter;
+    const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const filter_obj = {user_uid: uuid};
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
-    const file_count = await db_api.getRecords('files', filter_obj, true);
-
-    files = JSON.parse(JSON.stringify(files));
+    const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
 
     res.send({
         files: files,
@@ -1078,9 +1078,6 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
             await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
         } else if (is_playlist) {
             await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
-        } else if (type === 'subscription') {
-            // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
-            //          time they are requested from the subscription directory.
         } else {
             // error
             success = false;
@@ -1095,7 +1092,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
     });
 });
 
-app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
+app.post('/api/incrementViewCount', async (req, res) => {
     let file_uid = req.body.file_uid;
     let sub_id = req.body.sub_id;
     let uuid = req.body.uuid;
@@ -1230,12 +1227,9 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
-    let file = req.body.file;
     let file_uid = req.body.file_uid;
-    let sub = req.body.sub;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
+    let success = await db_api.deleteFile(file_uid, deleteForever);
 
     if (success) {
         res.send({
@@ -1268,7 +1262,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
     subscription = JSON.parse(JSON.stringify(subscription));
 
     // get sub videos
-    if (subscription.name && !subscription.streamingOnly) {
+    if (subscription.name) {
         var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
         subscription['videos'] = parsed_files;
         // loop through files for extra processing
@@ -1278,19 +1272,6 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
         }
 
-        res.send({
-            subscription: subscription,
-            files: parsed_files
-        });
-    } else if (subscription.name && subscription.streamingOnly) {
-        // return list of videos
-        let parsed_files = [];
-        if (subscription.videos) {
-            for (let i = 0; i < subscription.videos.length; i++) {
-                const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
-            }
-        }
         res.send({
             subscription: subscription,
             files: parsed_files
@@ -1335,9 +1316,8 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
 app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
-    let type = req.body.type;
 
-    const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1365,7 +1345,6 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     res.send({
         playlist: playlist,
         file_objs: file_objs,
-        type: playlist && playlist.type,
         success: !!playlist
     });
 });
@@ -1431,10 +1410,9 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
 app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const uid = req.body.uid;
     const blacklistMode = req.body.blacklistMode;
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     let wasDeleted = false;
-    wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
+    wasDeleted = await db_api.deleteFile(uid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1466,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
+        wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1527,20 +1505,69 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     });
 });
 
+app.post('/api/getArchives', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id;
+    const filter_obj = {user_uid: uuid, sub_id: sub_id};
+    const type = req.body.type;
+
+    // we do this for file types because if type is null, that means get files of all types
+    if (type) filter_obj['type'] = type;
+
+    const archives = await db_api.getRecords('archives', filter_obj);
+
+    res.send({
+        archives: archives
+    });
+});
+
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    let sub = req.body.sub;
-    let archive_dir = sub.archive;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
 
-    let full_archive_path = path.join(archive_dir, 'archive.txt');
+    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
 
-    if (await fs.pathExists(full_archive_path)) {
-        res.sendFile(full_archive_path);
+    if (archive_text !== null && archive_text !== undefined) {
+        res.setHeader('Content-type', "application/octet-stream");
+        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
+        res.send(archive_text);
     } else {
-        res.sendStatus(404);
+        res.sendStatus(400);
     }
 
 });
 
+app.post('/api/importArchive', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archive = req.body.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
+
+    const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
+
+    const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
+
+    res.send({
+        success: !!imported_count,
+        imported_count: imported_count
+    });
+});
+
+app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archives = req.body.archives;
+
+    let success = true;
+    for (const archive of archives) {
+        success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
+    }
+
+    res.send({
+        success: success
+    });
+});
+
 var upload_multer = multer({ dest: __dirname + '/appdata/' });
 app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
     const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1612,7 +1639,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
-        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
+        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1732,8 +1759,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
 
 app.post('/api/restartDownload', optionalJwt, async (req, res) => {
     const download_uid = req.body.download_uid;
-    const success = await downloader_api.restartDownload(download_uid);
-    res.send({success: success});
+    const new_download = await downloader_api.restartDownload(download_uid);
+    res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
 });
 
 app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1810,6 +1837,15 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
+    const task_key = req.body.task_key;
+    const new_options = req.body.new_options;
+  
+    const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
+
+    res.send({success: success});
+});
+
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
     fs.ensureDirSync(backup_dir);
@@ -1890,9 +1926,34 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
 
 // user authentication
 
-app.post('/api/auth/register'
-        , optionalJwt
-        , auth_api.registerUser);
+app.post('/api/auth/register', optionalJwt, async (req, res) => {
+    const userid = req.body.userid;
+    const username = req.body.username;
+    const plaintextPassword = req.body.password;
+
+    if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
+        logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
+        res.sendStatus(409);
+        return;
+    }
+
+    if (plaintextPassword === "") {
+        logger.error(`Registration failed for user ${userid}. A password must be provided.`);
+        res.sendStatus(409);
+        return;
+    }
+  
+    const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
+  
+    if (!new_user) {
+      res.sendStatus(409);
+      return;
+    }
+  
+    res.send({
+      user: new_user
+    });
+});
 app.post('/api/auth/login'
         , auth_api.passport.authenticate(['local', 'ldapauth'], {})
         , auth_api.generateJWT
@@ -1944,18 +2005,7 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
 app.post('/api/deleteUser', optionalJwt, async (req, res) => {
     let uid = req.body.uid;
     try {
-        let success = false;
-        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const user_folder = path.join(__dirname, usersFileFolder, uid);
-        const user_db_obj = await db_api.getRecord('users', {uid: uid});
-        if (user_db_obj) {
-            // user exists, let's delete
-            await fs.remove(user_folder);
-            await db_api.removeRecord('users', {uid: uid});
-            success = true;
-        } else {
-            logger.error(`Could not find user with uid ${uid}`);
-        }
+        const success = await auth_api.deleteUser(uid);
         res.send({success: success});
     } catch (err) {
         logger.error(err);
@@ -1993,6 +2043,93 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+// notifications
+
+app.post('/api/getNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
+
+    res.send({notifications: notifications});
+});
+
+// set notifications to read
+app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
+    const uid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeRecord('notifications', {uid: uid});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
+
+    res.send({success: success});
+});
+
+// rss feed
+
+app.get('/api/rss', async function (req, res) {
+    if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
+        logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
+        res.sendStatus(403);
+        return;
+    }
+
+    // these are returned
+    const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
+    const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
+    const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
+    const file_type_filter = req.query.file_type_filter;
+    const favorite_filter = req.query.favorite_filter === 'true';
+    const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
+    const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
+
+    const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+
+    const feed = new Feed({
+            title: 'Downloads',
+            description: 'YoutubeDL-Material downloads',
+            id: utils.getBaseURL(),
+            link: utils.getBaseURL(),
+            image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
+            favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
+            generator: 'YoutubeDL-Material'
+    });
+
+    files.forEach(file => {
+        feed.addItem({
+            title: file.title,
+            link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
+            description: file.description,
+            author: [
+                {
+                    name: file.uploader,
+                    link: file.url
+                }
+            ],
+            contributor: [],
+            date: file.timestamp,
+            // https://stackoverflow.com/a/45415677/8088021
+            image: file.thumbnailURL.replace('&', '&amp;')
+        });
+      });
+    res.send(feed.rss2());
+});
+
+// web server
+
 app.use(function(req, res, next) {
     //if the request is not html then move along
     var accept = req.accepts('html', 'json', 'xml');

backend/appdata/default.json

@@ -23,7 +23,12 @@
       "download_only_mode": false,
       "allow_autoplay": true,
       "enable_downloads_manager": true,
-      "allow_playlist_categorization": true
+      "allow_playlist_categorization": true,
+      "force_autoplay": false,
+      "enable_notifications": true,
+      "enable_all_notifications": true,
+      "allowed_notification_types": [],
+      "enable_rss_feed": false
     },
     "API": {
       "use_API_key": false,
@@ -31,10 +36,21 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
-      "generate_NFO_files": false
+      "generate_NFO_files": false,
+      "use_ntfy_API": false,
+      "ntfy_topic_URL": "",
+      "use_gotify_API": false,
+      "gotify_server_URL": "",
+      "gotify_app_token": "",
+      "use_telegram_API": false,
+      "telegram_bot_token": "",
+      "telegram_chat_id": "",
+      "webhook_URL": "",
+      "discord_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",
@@ -63,7 +79,7 @@
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
-      "default_downloader": "youtube-dl",
+      "default_downloader": "yt-dlp",
       "use_default_downloading_agent": true,
       "custom_downloading_agent": "",
       "multi_user_mode": false,

backend/archive.js

@@ -0,0 +1,91 @@
+const path = require('path');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+
+const db_api = require('./db');
+
+exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
+    const filter = {user_uid: user_uid, sub_id: sub_id};
+    if (type) filter['type'] = type;
+    const archive_items = await db_api.getRecords('archives', filter);
+    const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
+    return archive_item_lines.join('\n');
+}
+
+exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
+    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
+    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
+    return success;
+}
+
+exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return success;
+}
+
+exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
+    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return !!archive_item;
+}
+
+exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
+    let archive_import_count = 0;
+    const lines = archive_text.split('\n');
+    for (let line of lines) {
+        const archive_line_parts = line.trim().split(' ');
+        // should just be the extractor and the video ID
+        if (archive_line_parts.length !== 2) {
+            continue;
+        }
+
+        const extractor = archive_line_parts[0];
+        const id        = archive_line_parts[1];
+        if (!extractor || !id) continue;
+
+        // we can't do a bulk write because we need to avoid duplicate archive items existing in db
+
+        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
+        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
+        archive_import_count++;
+    }
+    return archive_import_count;
+}
+
+exports.importArchives = async () => {
+    const imported_archives = [];
+    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        if (!dir_to_check['archive_path']) continue;
+
+        const files_to_import = [
+            path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
+            path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
+        ]
+
+        for (const file_to_import of files_to_import) {
+            const file_exists = await fs.pathExists(file_to_import);
+            if (!file_exists) continue;
+
+            const archive_text = await fs.readFile(file_to_import, 'utf8');
+            await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
+            imported_archives.push(file_to_import);
+        }
+    }
+    return imported_archives;
+}
+
+const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
+    return {
+        extractor: extractor,
+        id: id,
+        type: type,
+        title: title,
+        user_uid: user_uid ? user_uid : null,
+        sub_id: sub_id ? sub_id : null,
+        timestamp: Date.now() / 1000,
+        uid: uuid()
+    }
+}

backend/authentication/auth.js

@@ -6,6 +6,8 @@ const db_api = require('../db');
 const jwt = require('jsonwebtoken');
 const { uuid } = require('uuidv4');
 const bcrypt = require('bcryptjs');
+const fs = require('fs-extra');
+const path = require('path');
 
 var LocalStrategy = require('passport-local').Strategy;
 var LdapStrategy = require('passport-ldapauth');
@@ -16,7 +18,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
 let SERVER_SECRET = null;
 let JWT_EXPIRATION = null;
 let opts = null;
-let saltRounds = null;
+let saltRounds = 10;
 
 exports.initialize = function () {
   /*************************
@@ -31,9 +33,14 @@ exports.initialize = function () {
       });
   }
 
-  saltRounds = 10;
-
+  // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
+  if (!(+JWT_EXPIRATION)) {
+    logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
+    JWT_EXPIRATION = 86400;
+  } else {
+    JWT_EXPIRATION = +JWT_EXPIRATION;
+  }
 
   SERVER_SECRET = null;
   if (db_api.users_db.get('jwt_secret').value()) {
@@ -106,55 +113,41 @@ exports.passport.deserializeUser(function(user, done) {
 /***************************************
  * Register user with hashed password
  **************************************/
-exports.registerUser = async function(req, res) {
-  var userid = req.body.userid;
-  var username = req.body.username;
-  var plaintextPassword = req.body.password;
 
-  if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
-    res.sendStatus(409);
-    logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
-    return;
+exports.registerUser = async (userid, username, plaintextPassword) => {
+  const hash = await bcrypt.hash(plaintextPassword, saltRounds);
+  const new_user = generateUserObject(userid, username, hash);
+  // check if user exists
+  if (await db_api.getRecord('users', {uid: userid})) {
+    // user id is taken!
+    logger.error('Registration failed: UID is already taken!');
+    return null;
+  } else if (await db_api.getRecord('users', {name: username})) {
+      // user name is taken!
+      logger.error('Registration failed: User name is already taken!');
+      return null;
+  } else {
+    // add to db
+    await db_api.insertRecordIntoTable('users', new_user);
+    logger.verbose(`New user created: ${new_user.name}`);
+    return new_user;
   }
+}
 
-  if (plaintextPassword === "") {
-    res.sendStatus(400);
-    logger.error(`Registration failed for user ${userid}. A password must be provided.`);
-    return;
+exports.deleteUser = async (uid) => {
+  let success = false;
+  let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+  const user_folder = path.join(__dirname, usersFileFolder, uid);
+  const user_db_obj = await db_api.getRecord('users', {uid: uid});
+  if (user_db_obj) {
+      // user exists, let's delete
+      await fs.remove(user_folder);
+      await db_api.removeRecord('users', {uid: uid});
+      success = true;
+  } else {
+      logger.error(`Could not find user with uid ${uid}`);
   }
-
-  bcrypt.hash(plaintextPassword, saltRounds)
-    .then(async function(hash) {
-      let new_user = generateUserObject(userid, username, hash);
-      // check if user exists
-      if (await db_api.getRecord('users', {uid: userid})) {
-        // user id is taken!
-        logger.error('Registration failed: UID is already taken!');
-        res.status(409).send('UID is already taken!');
-      } else if (await db_api.getRecord('users', {name: username})) {
-          // user name is taken!
-          logger.error('Registration failed: User name is already taken!');
-          res.status(409).send('User name is already taken!');
-      } else {
-        // add to db
-        await db_api.insertRecordIntoTable('users', new_user);
-        logger.verbose(`New user created: ${new_user.name}`);
-        res.send({
-          user: new_user
-        });
-      }
-    })
-    .then(function(result) {
-
-    })
-    .catch(function(err) {
-      logger.error(err);
-      if( err.code == 'ER_DUP_ENTRY' ) {
-        res.status(409).send('UserId already taken');
-      } else {
-        res.sendStatus(409);
-      }
-    });
+  return success;
 }
 
 /***************************************
@@ -319,7 +312,7 @@ exports.getUserVideos = async function(user_uid, type) {
 }
 
 exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
-  let file = await db_api.getRecord('files', {file_uid: file_uid});
+  let file = await db_api.getRecord('files', {uid: file_uid});
 
   // prevent unauthorized users from accessing the file info
   if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -361,7 +354,6 @@ exports.userHasPermission = async function(user_uid, permission) {
     logger.error('Invalid role ' + role);
     return false;
   }
-  const role_permissions = (await db_api.getRecords('roles'))['permissions'];
 
   const user_has_explicit_permission = user_obj['permissions'].includes(permission);
   const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -376,7 +368,8 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 
   // no overrides, let's check if the role has the permission
-  if (role_permissions.includes(permission)) {
+  const role_has_permission = await exports.roleHasPermissions(role, permission);
+  if (role_has_permission) {
     return true;
   } else {
     logger.verbose(`User ${user_uid} failed to get permission ${permission}`);
@@ -384,6 +377,16 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 }
 
+exports.roleHasPermissions = async function(role, permission) {
+  const role_obj = await db_api.getRecord('roles', {key: role})
+  if (!role) {
+    logger.error(`Role ${role} does not exist!`);
+  }
+  const role_permissions = role_obj['permissions'];
+  if (role_permissions && role_permissions.includes(permission)) return true;
+  else return false;
+}
+
 exports.userPermissions = async function(user_uid) {
   let user_permissions = [];
   const user_obj = await db_api.getRecord('users', ({uid: user_uid}));

backend/config.js

@@ -185,7 +185,6 @@ const DEFAULT_CONFIG = {
         "default_file_output": "",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false,
         "include_thumbnail": true,
         "include_metadata": true,
         "max_concurrent_downloads": 5,
@@ -196,20 +195,32 @@ const DEFAULT_CONFIG = {
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_autoplay": true,
+        "force_autoplay": false,
         "enable_downloads_manager": true,
-        "allow_playlist_categorization": true
+        "allow_playlist_categorization": true,
+        "enable_notifications": true,
+        "enable_all_notifications": true,
+        "allowed_notification_types": [],
+        "enable_rss_feed": false,
       },
       "API": {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
-        "use_twitch_API": false,
-        "twitch_API_key": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
-        "generate_NFO_files": false
+        "generate_NFO_files": false,
+        "use_ntfy_API": false,
+        "ntfy_topic_URL": "",
+        "use_gotify_API": false,
+        "gotify_server_URL": "",
+        "gotify_app_token": "",
+        "use_telegram_API": false,
+        "telegram_bot_token": "",
+        "telegram_chat_id": "",
+        "webhook_URL": "",
+        "discord_webhook_URL": ""
       },
       "Themes": {
         "default_theme": "default",
@@ -238,7 +249,7 @@ const DEFAULT_CONFIG = {
         "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
       },
       "Advanced": {
-        "default_downloader": "youtube-dl",
+        "default_downloader": "yt-dlp",
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",
         "multi_user_mode": false,

backend/consts.js

@@ -30,10 +30,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'
     },
-    'ytdl_safe_download_override': {
-        'key': 'ytdl_safe_download_override',
-        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
-    },
     'ytdl_include_thumbnail': {
         'key': 'ytdl_include_thumbnail',
         'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -68,9 +64,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_autoplay': {
-        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+    'ytdl_force_autoplay': {
+        'key': 'ytdl_force_autoplay',
+        'path': 'YoutubeDLMaterial.Extra.force_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -80,6 +76,22 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_allow_playlist_categorization',
         'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
+    'ytdl_enable_notifications': {
+        'key': 'ytdl_enable_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_notifications'
+    },
+    'ytdl_enable_all_notifications': {
+        'key': 'ytdl_enable_all_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
+    },
+    'ytdl_allowed_notification_types': {
+        'key': 'ytdl_allowed_notification_types',
+        'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
+    },
+    'ytdl_enable_rss_feed': {
+        'key': 'ytdl_enable_rss_feed',
+        'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
+    },
 
     // API
     'ytdl_use_api_key': {
@@ -98,14 +110,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
-    'ytdl_use_twitch_api': {
-        'key': 'ytdl_use_twitch_api',
-        'path': 'YoutubeDLMaterial.API.use_twitch_API'
-    },
-    'ytdl_twitch_api_key': {
-        'key': 'ytdl_twitch_api_key',
-        'path': 'YoutubeDLMaterial.API.twitch_API_key'
-    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -118,6 +122,46 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_generate_nfo_files',
         'path': 'YoutubeDLMaterial.API.generate_NFO_files'
     },
+    'ytdl_use_ntfy_API': {
+        'key': 'ytdl_use_ntfy_API',
+        'path': 'YoutubeDLMaterial.API.use_ntfy_API'
+    },
+    'ytdl_ntfy_topic_url': {
+        'key': 'ytdl_ntfy_topic_url',
+        'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
+    },
+    'ytdl_use_gotify_API': {
+        'key': 'ytdl_use_gotify_API',
+        'path': 'YoutubeDLMaterial.API.use_gotify_API'
+    },
+    'ytdl_gotify_server_url': {
+        'key': 'ytdl_gotify_server_url',
+        'path': 'YoutubeDLMaterial.API.gotify_server_URL'
+    },
+    'ytdl_gotify_app_token': {
+        'key': 'ytdl_gotify_app_token',
+        'path': 'YoutubeDLMaterial.API.gotify_app_token'
+    },
+    'ytdl_use_telegram_API': {
+        'key': 'ytdl_use_telegram_API',
+        'path': 'YoutubeDLMaterial.API.use_telegram_API'
+    },
+    'ytdl_telegram_bot_token': {
+        'key': 'ytdl_telegram_bot_token',
+        'path': 'YoutubeDLMaterial.API.telegram_bot_token'
+    },
+    'ytdl_telegram_chat_id': {
+        'key': 'ytdl_telegram_chat_id',
+        'path': 'YoutubeDLMaterial.API.telegram_chat_id'
+    },
+    'ytdl_webhook_url': {
+        'key': 'ytdl_webhook_url',
+        'path': 'YoutubeDLMaterial.API.webhook_URL'
+    },
+    'ytdl_discord_webhook_url': {
+        'key': 'ytdl_discord_webhook_url',
+        'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
+    },
 
 
     // Themes
@@ -217,7 +261,8 @@ exports.AVAILABLE_PERMISSIONS = [
     'subscriptions',
     'sharing',
     'advanced_download',
-    'downloads_manager'
+    'downloads_manager',
+    'tasks_manager'
 ];
 
 exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
@@ -301,4 +346,6 @@ const YTDL_ARGS_WITH_VALUES = [
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
-exports.CURRENT_VERSION = 'v4.2';
+exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
+
+exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -2,6 +2,7 @@ var fs = require('fs-extra')
 var path = require('path')
 const { MongoClient } = require("mongodb");
 const { uuid } = require('uuidv4');
+const _ = require('lodash');
 
 const config_api = require('./config');
 var utils = require('./utils')
@@ -58,6 +59,13 @@ const tables = {
         name: 'tasks',
         primary_key: 'key'
     },
+    notifications: {
+        name: 'notifications',
+        primary_key: 'uid'
+    },
+    archives: {
+        name: 'archives'
+    },
     test: {
         name: 'test'
     }
@@ -148,6 +156,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
                 await database.collection(table).createIndex(text_search);
             }
         });
+        using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
         return true;
     } catch(err) {
         logger.error(err);
@@ -198,7 +207,7 @@ async function registerFileDBManual(file_object) {
     path_object = path.parse(file_object['path']);
     file_object['path'] = path.format(path_object);
 
-    exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+    await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
 
     return file_object;
 }
@@ -252,13 +261,16 @@ exports.getFileDirectoriesAndDBs = async () => {
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'audio'),
                 user_uid: user.uid,
-                type: 'audio'
+                type: 'audio',
+                archive_path: utils.getArchiveFolder('audio', user.uid)
             });
 
             // add user's video dir to check list
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'video'),
-                type: 'video'
+                user_uid: user.uid,
+                type: 'video',
+                archive_path: utils.getArchiveFolder('video', user.uid)
             });
         }
     } else {
@@ -268,13 +280,15 @@ exports.getFileDirectoriesAndDBs = async () => {
         // add audio dir to check list
         dirs_to_check.push({
             basePath: audioFolderPath,
-            type: 'audio'
+            type: 'audio',
+            archive_path: utils.getArchiveFolder('audio')
         });
 
         // add video dir to check list
         dirs_to_check.push({
             basePath: videoFolderPath,
-            type: 'video'
+            type: 'video',
+            archive_path: utils.getArchiveFolder('video')
         });
     }
 
@@ -295,7 +309,8 @@ exports.getFileDirectoriesAndDBs = async () => {
                                       : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
             user_uid: subscription_to_check.user_uid,
             type: subscription_to_check.type,
-            sub_id: subscription_to_check['id']
+            sub_id: subscription_to_check['id'],
+            archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
         });
     }
 
@@ -350,14 +365,14 @@ exports.addMetadataPropertyToDB = async (property_key) => {
             }
         }
 
-        return await exports.bulkUpdateRecords('files', 'uid', update_obj);
+        return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
     } catch(err) {
         logger.error(err);
         return false;
     }
 }
 
-exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
     const first_video = await exports.getVideo(uids[0]);
     const thumbnailToUse = first_video['thumbnailURL'];
     
@@ -366,7 +381,6 @@ exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
         uids: uids,
         id: uuid(),
         thumbnailURL: thumbnailToUse,
-        type: type,
         registered: Date.now(),
         randomize_order: false
     };
@@ -448,8 +462,8 @@ exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null)
     return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
 }
 
-exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid, uuid);
+exports.deleteFile = async (uid, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid);
     const type = file_obj.isAudio ? 'audio' : 'video';
     const folderPath = path.dirname(file_obj.path);
     const ext = type === 'audio' ? 'mp3' : 'mp4';
@@ -495,22 +509,21 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
+        // get id/extractor from JSON
 
-        // get ID from JSON
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedID = null;
+        let retrievedExtractor = null;
+        if (info_json) {
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
 
-        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let id = null;
-        if (jsonobj) id = jsonobj.id;
-
-        // use subscriptions API to remove video from the archive file, and write it to the blacklist
-        if (await fs.pathExists(archive_path)) {
-            const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
-            if (blacklistMode && line) await writeToBlacklist(type, line);
-        } else {
-            logger.info('Could not find archive file for audio files. Creating...');
-            await fs.close(await fs.open(archive_path, 'w'));
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        if (!blacklistMode) {
+            // workaround until a files_api is created (using archive_api would make a circular dependency)
+            await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
+            // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
         }
     }
 
@@ -542,8 +555,32 @@ exports.getVideo = async (file_uid) => {
     return await exports.getRecord('files', {uid: file_uid});
 }
 
-exports.getFiles = async (uuid = null) => {
-    return await exports.getRecords('files', {user_uid: uuid});
+exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (favorite_filter) {
+        filter_obj['favorite'] = true;
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
+    const file_count = await exports.getRecords('files', filter_obj, true);
+
+    return {files, file_count};
 }
 
 exports.setVideoProperty = async (file_uid, assignment_obj) => {
@@ -558,7 +595,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
 exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
     // local db override
     if (using_local_db) {
-        if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
+        if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
         local_db.get(table).push(doc).write();
         return true;
     }
@@ -661,9 +698,15 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 
 // Update
 
-exports.updateRecord = async (table, filter_obj, update_obj) => {
+exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
     // local db override
     if (using_local_db) {
+        if (nested_mode) {
+            // if object is nested we need to handle it differently
+            update_obj = utils.convertFlatObjectToNestedObject(update_obj);
+            exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
+            return true;
+        }
         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
@@ -677,7 +720,14 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
+            const props_to_update = Object.keys(update_obj);
+            for (let i = 0; i < props_to_update.length; i++) {
+                const prop_to_update = props_to_update[i];
+                const prop_value = update_obj[prop_to_update];
+                record[prop_to_update] = prop_value;
+            }
+        }).write();
         return true;
     }
 
@@ -685,7 +735,19 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
     return !!(output['result']['ok']);
 }
 
-exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
+exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
+    // local db override
+    if (using_local_db) {
+        const props_to_remove = Object.keys(remove_obj);
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
+        return true;
+    }
+
+    const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
+    return !!(output['result']['ok']);
+}
+
+exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
     // local db override
     if (using_local_db) {
         local_db.get(table).each((record) => {
@@ -933,6 +995,7 @@ exports.importJSONToDB = async (db_json, users_json) => {
 const createFilesRecords = (files, subscriptions) => {
     for (let i = 0; i < subscriptions.length; i++) {
         const subscription = subscriptions[i];
+        if (!subscription['videos']) continue;
         subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
         files = files.concat(subscriptions[i]['videos']);
     }
@@ -993,7 +1056,7 @@ exports.backupDB = async () => {
     const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
     const path_to_backups = path.join(backup_dir, backup_file_name);
 
-    logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
+    logger.info(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
 
     const table_to_records = {};
     for (let i = 0; i < tables_list.length; i++) {
@@ -1040,10 +1103,11 @@ exports.transferDB = async (local_to_remote) => {
         table_to_records[table] = await exports.getRecords(table);
     }
 
+    logger.info('Backup up DB...');
+    await exports.backupDB(); // should backup always
+
     using_local_db = !local_to_remote;
     if (local_to_remote) {
-        logger.debug('Backup up DB...');
-        await exports.backupDB();
         const db_connected = await exports.connectToDB(5, true);
         if (!db_connected) {
             logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1097,6 +1161,14 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
                     } else if ('$ne' in filter_prop_value) {
                         filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
+                    } else if ('$lt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
+                    } else if ('$gt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
+                    } else if ('$lte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
+                    } else if ('$gte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
                     }
                 } else {
                     // handle case of nested property check
@@ -1112,14 +1184,7 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     return return_val;
 }
 
-// archive helper functions
-
-async function writeToBlacklist(type, line) {
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}
+// should only be used for tests
+exports.setLocalDBMode = (mode) => {
+    using_local_db = mode;
+}

backend/downloader.js

@@ -1,7 +1,6 @@
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const path = require('path');
-const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
@@ -14,12 +13,12 @@ const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
+const notifications_api = require('./notifications');
+const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
 if (db_api.database_initialized) {
     setupDownloads();
 } else {
@@ -28,7 +27,26 @@ if (db_api.database_initialized) {
     });
 }
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+/*
+
+This file handles all the downloading functionality.
+
+To download a file, we go through 4 steps. Here they are with their respective index & function:
+
+0: Create the download
+ - createDownload()
+1: Get info for the download (we need this step for categories and archive functionality)
+ - collectInfo()
+2: Download the file
+ - downloadQueuedFile()
+3: Complete
+ - N/A
+
+We use checkDownloads() to move downloads through the steps and call their respective functions.
+
+*/
+
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -37,6 +55,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             user_uid: user_uid,
             sub_id: sub_id,
             sub_name: sub_name,
+            prefetched_info: prefetched_info,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -85,10 +104,10 @@ exports.resumeDownload = async (download_uid) => {
 exports.restartDownload = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     await exports.clearDownload(download_uid);
-    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
     
     should_check_downloads = true;
-    return success;
+    return new_download;
 }
 
 exports.cancelDownload = async (download_uid) => {
@@ -107,9 +126,10 @@ exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download_uid, error_message) {
-    if (!download_uid) return;
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+async function handleDownloadError(download, error_message, error_type = null) {
+    if (!download || !download['uid']) return;
+    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
+    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
 async function setupDownloads() {
@@ -155,6 +175,13 @@ async function checkDownloads() {
         if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
 
         if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            if (waiting_download['sub_id']) {
+                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
+                if (sub_missing) {
+                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    continue;
+                }
+            }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
@@ -187,13 +214,27 @@ async function collectInfo(download_uid) {
     let args = await exports.generateArgs(url, type, options, download['user_uid']);
 
     // get video info prior to download
-    let info = await exports.getVideoInfoByURL(url, args, download_uid);
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
     if (!info) {
         // info failed, error presumably already recorded
         return;
     }
 
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
+        if (exists_in_archive) {
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            logger.warn(error);
+            if (download_uid) {
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error, 'exists_in_archive');
+                return;
+            }
+        }
+    }
+
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -207,7 +248,7 @@ async function collectInfo(download_uid) {
         info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
-    download['category'] = category;
+    const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
 
     // setup info required to calculate download progress
 
@@ -229,7 +270,9 @@ async function collectInfo(download_uid) {
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    category: stripped_category,
+                                                                    prefetched_info: null
                                                                 });
 }
 
@@ -242,6 +285,7 @@ async function downloadQueuedFile(download_uid) {
     return new Promise(async resolve => {
         const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
         await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
 
         const url = download['url'];
@@ -249,9 +293,11 @@ async function downloadQueuedFile(download_uid) {
         const options = download['options'];
         const args = download['args'];
         const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
         if (options.customFileFolderPath) {
             fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
         }
         fs.ensureDirSync(fileFolderPath);
 
@@ -268,14 +314,14 @@ async function downloadQueuedFile(download_uid) {
             clearInterval(download_checker);
             if (err) {
                 logger.error(err.stderr);
-                await handleDownloadError(download_uid, err.stderr);
+                await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_output');
                     logger.warn(error_message);
                     resolve(false);
                     return;
@@ -284,7 +330,10 @@ async function downloadQueuedFile(download_uid) {
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
-                        output_json = JSON.parse(output[i]);
+                        // we have to do this because sometimes there will be leading characters before the actual json
+                        const start_idx = output[i].indexOf('{"');
+                        const clean_output = output[i].slice(start_idx, output[i].length);
+                        output_json = JSON.parse(clean_output);
                     } catch(e) {
                         output_json = null;
                     }
@@ -301,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -337,15 +386,12 @@ async function downloadQueuedFile(download_uid) {
                     // registers file in DB
                     const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
-                    file_objs.push(file_obj);
-                }
+                    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
 
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
+                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
+
+                    file_objs.push(file_obj);
                 }
 
                 let container = null;
@@ -353,13 +399,13 @@ async function downloadQueuedFile(download_uid) {
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
                 } else {
                     const error_message = 'Downloaded file failed to result in metadata object.';
                     logger.error(error_message);
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_metadata');
                 }
 
                 const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -373,15 +419,27 @@ async function downloadQueuedFile(download_uid) {
 // helper functions
 
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+
+    if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
+        logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
+    }
+
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
 
     const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
     const globalArgs = config_api.getConfigItem('ytdl_custom_args');
     const useCookies = config_api.getConfigItem('ytdl_use_cookies');
     const is_audio = type === 'audio';
 
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
 
     if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
 
@@ -391,6 +449,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 
     // video-specific args
     const selectedHeight = options.selectedHeight;
+    const maxHeight = options.maxHeight;
+    const heightParam = selectedHeight || maxHeight;
 
     // audio-specific args
     const maxBitrate = options.maxBitrate;
@@ -404,8 +464,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     if (!is_audio && !is_youtube) {
         // tiktok videos fail when using the default format
         qualityPath = null;
-    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
-        qualityPath = ['-f', 'bestvideo+bestaudio']
     }
 
     if (customArgs) {
@@ -413,8 +471,9 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     } else {
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
-        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+        } else if (heightParam && heightParam !== '' && !is_audio) {
+            const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
+            qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -451,28 +510,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -496,9 +533,11 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.push('-r', rate_limit);
         }
         
-        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
         if (default_downloader === 'yt-dlp') {
-            downloadConfig.push('--no-clean-infojson');
+            downloadConfig = utils.filterArgs(downloadConfig, ['--print-json']);
+
+            // in yt-dlp -j --no-simulate is preferable
+            downloadConfig.push('--no-clean-info-json', '-j', '--no-simulate');
         }
 
     }
@@ -506,14 +545,15 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     // filter out incompatible args
     downloadConfig = filterArgs(downloadConfig, is_audio);
 
-    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
     return downloadConfig;
 }
 
 exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
-        const new_args = [...args];
+        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...temp_args];
 
         const archiveArgIndex = new_args.indexOf('--download-archive');
         if (archiveArgIndex !== -1) {
@@ -545,7 +585,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                     const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
                     logger.error(error);
                     if (download_uid) {
-                        await handleDownloadError(download_uid, error);
+                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download, error, 'parse_failed');
                     }
                     resolve(null);
                 }
@@ -554,7 +595,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                 if (err.stderr) error_message += `\n\n${err.stderr}`;
                 logger.error(error_message);
                 if (download_uid) {
-                    await handleDownloadError(download_uid, error_message);
+                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
                 }
                 resolve(null);
             }
@@ -565,8 +607,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
 function filterArgs(args, isAudio) {
     const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
     const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
-    return args.filter(x => !args_to_remove.includes(x));
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
 }
 
 async function checkDownloadPercent(download_uid) {
@@ -621,13 +662,3 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join(archivePath);
-    }
-}

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="pm2-runtime pm2.config.js"
+CMD="npm start"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then
@@ -10,7 +10,7 @@ fi
 
 # chown current working directory to current user
 if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
   exec gosu "$UID:$GID" "$0" "$@"
 fi
 

backend/fix-scripts/001-fix_download_permissions.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M
 # Date: 2022-05-03
@@ -6,8 +6,7 @@
 # If you want to run this script on a bare-metal installation instead of within Docker
 # make sure that the paths configured below match your paths! (it's wise to use the full paths)
 # USAGE: within your container's bash shell:
-# chmod -R +x ./fix-scripts/
-# ./fix-scripts/001-fix_download_permissions.sh
+# ./fix-scripts/<name of fix-script>
 
 # User defines / Docker env defaults
 PATH_SUBS=/app/subscriptions

backend/fix-scripts/002-fix_dupes_per_archive_file.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-09
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines (NO TRAILING SLASHES) / Docker env defaults
+PATH_SUBSARCHIVE=/app/subscriptions/archives
+PATH_ONEOFFARCHIVE=/app/appdata/archives
+
+# Backup paths (substitute with your personal preference if you like)
+PATH_SUBSARCHIVEBKP=$PATH_SUBSARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+PATH_ONEOFFARCHIVEBKP=$PATH_ONEOFFARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+
+
+# Define Colors for TUI
+yellow=$(tput setaf 3)
+normal=$(tput sgr0)
+
+tput civis                                                          # hide the cursor
+
+clear -x
+printf "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "Welcome to the INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M."
+printf "\nThis script will cycle through the archive files in the folders mentioned"
+printf "\nbelow and remove within each archive the dupe entries. (compact them)"
+printf "\nDuring some older builds of YTDL-M the archives could receive dupe"
+printf "\nentries and blow up in size, sometimes causing conflicts with download management."
+printf '\n%*s' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBSARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found Subscriptions archive directory at ${PATH_SUBSARCHIVE}"
+[ -d $PATH_ONEOFFARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found one-off archive directory at ${PATH_ONEOFFARCHIVE}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    printf "\n\n  Couldn't find any archive location path! \n\nPlease edit this script to configure!"
+    tput cnorm
+    exit 2;;
+  2)
+    printf "\n\n  Found all archive locations. \n\nProceed? (Y/N)";;
+  *)
+    printf "\n\n  Only found ${i} out of 2 archive locations! Something about this script's config must be wrong. \n\nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg          # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    printf "\n\nRunning jobs now... (this may take a while)\n"
+
+    printf "\nBacking up directories...\n"
+
+    chars="⣾⣽⣻⢿⡿⣟⣯⣷"
+    cp -R $PATH_SUBSARCHIVE $PATH_SUBSARCHIVEBKP &
+    PID=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.15
+    done
+    [ -d $PATH_SUBSARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_SUBSARCHIVE} to ${PATH_SUBSARCHIVEBKP} ($(du -sh $PATH_SUBSARCHIVEBKP | cut -f1))\n"
+
+    cp -R $PATH_ONEOFFARCHIVE $PATH_ONEOFFARCHIVEBKP &
+    PID2=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID2 ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.1
+    done
+    [ -d $PATH_ONEOFFARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_ONEOFFARCHIVE} to ${PATH_ONEOFFARCHIVEBKP} ($(du -sh $PATH_ONEOFFARCHIVEBKP | cut -f1))\n"
+
+
+    printf "\nCompacting files...\n"
+
+    tmpfile=$(mktemp) &&
+
+    [ -d $PATH_SUBSARCHIVE ] &&
+    find $PATH_SUBSARCHIVE -name '*.txt' -print0 | while read -d $'\0' file # Set delimiter to null because we want to catch all possible filenames (WE CANNOT CHANGE IFS HERE) - https://stackoverflow.com/a/15931055
+    do
+        cp "$file" "$tmpfile"
+        { awk '!x[$0]++' "$tmpfile" > "$file"; } &                         # https://unix.stackexchange.com/questions/159695/how-does-awk-a0-work
+        PID3=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID3 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [[ "$AFTER" -ne "$BEFORE" ]]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action needed for file: ${file}\n"
+        fi
+    done
+
+    [ -d $PATH_ONEOFFARCHIVE ] &&
+    find $PATH_ONEOFFARCHIVE -name '*.txt' -print0 | while read -d $'\0' file
+    do
+        cp "$file" "$tmpfile" &
+        awk '!x[$0]++' "$tmpfile" > "$file" &
+        PID4=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID4 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [ "$BEFORE" -ne "$AFTER" ]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action ran for file: ${file}\n"
+        fi
+    done
+    tput cnorm                                                     # show the cursor
+    rm "$tmpfile"
+
+    printf "\n\n✔ Done."
+    printf "\nℹ Please keep in mind that you may still want to"
+    printf "\n  run corruption checks against your archives!\n\n"
+    exit
+else
+    tput cnorm
+    printf "\nOkay, bye.\n\n"
+    exit
+fi

backend/notifications.js

@@ -0,0 +1,187 @@
+const db_api = require('./db');
+const config_api = require('./config');
+const logger = require('./logger');
+const utils = require('./utils');
+const consts = require('./consts');
+
+const { uuid } = require('uuidv4');
+
+const fetch = require('node-fetch');
+const { gotify } = require("gotify");
+const TelegramBot = require('node-telegram-bot-api');
+const REST = require('@discordjs/rest').REST;
+const API = require('@discordjs/core').API;
+const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
+
+const NOTIFICATION_TYPE_TO_TITLE = {
+    task_finished: 'Task finished',
+    download_complete: 'Download complete',
+    download_error: 'Download error'
+}
+
+const NOTIFICATION_TYPE_TO_BODY = {
+    task_finished: (notification) => notification['data']['task_title'],
+    download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
+    download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
+}
+
+const NOTIFICATION_TYPE_TO_URL = {
+    task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
+    download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
+    download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
+}
+
+const NOTIFICATION_TYPE_TO_THUMBNAIL = {
+    task_finished: () => null,
+    download_complete: (notification) => notification['data']['file_thumbnail'],
+    download_error: () => null
+}
+
+exports.sendNotification = async (notification) => {
+    // info necessary if we are using 3rd party APIs
+    const type = notification['type'];
+
+    const data = {
+        title: NOTIFICATION_TYPE_TO_TITLE[type],
+        body: NOTIFICATION_TYPE_TO_BODY[type](notification),
+        type: type,
+        url: NOTIFICATION_TYPE_TO_URL[type](notification),
+        thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
+    }
+
+    if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
+        sendNtfyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
+        sendGotifyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
+        sendTelegramNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_webhook_url')) {
+        sendGenericNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
+        sendDiscordNotification(data);
+    }
+
+    await db_api.insertRecordIntoTable('notifications', notification);
+    return notification;
+}
+
+exports.sendTaskNotification = async (task_obj, confirmed) => {
+    if (!notificationEnabled('task_finished')) return;
+    // workaround for tasks which are user_uid agnostic
+    const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
+    await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
+    const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
+    const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadNotification = async (file, user_uid) => {
+    if (!notificationEnabled('download_complete')) return;
+    const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
+    const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
+    if (!notificationEnabled('download_error')) return;
+    const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
+    const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.createNotification = (type, actions, data, user_uid) => {
+    const notification = {
+        type: type,
+        actions: actions,
+        data: data,
+        user_uid: user_uid,
+        uid: uuid(),
+        read: false,
+        timestamp: Date.now()/1000
+    }
+    return notification;
+}
+
+function notificationEnabled(type) {
+    return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
+}
+
+function sendNtfyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to ntfy');
+    fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
+        method: 'POST',
+        body: body,
+        headers: {
+            'Title': title,
+            'Tags': type,
+            'Click': url,
+            'Attach': thumbnail
+        }
+    });
+}
+
+async function sendGotifyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to gotify');
+    await gotify({
+        server: config_api.getConfigItem('ytdl_gotify_server_url'),
+        app: config_api.getConfigItem('ytdl_gotify_app_token'),
+        title: title,
+        message: body,
+        tag: type,
+        priority: 5, // Keeping default from docs, may want to change this,
+        extras: {
+            "client::notification": {
+                click: { url: url },
+                bigImageUrl: thumbnail
+            }
+        }
+      });
+}
+
+async function sendTelegramNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to Telegram');
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    const bot = new TelegramBot(bot_token);
+    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+}
+
+async function sendDiscordNotification({body, title, type, url, thumbnail}) {
+    const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
+    const url_split = discord_webhook_url.split('webhooks/');
+    const [webhook_id, webhook_token] = url_split[1].split('/');
+    const rest = new REST({ version: '10' });
+    const api = new API(rest);
+    const embed = new EmbedBuilder()
+        .setTitle(title)
+        .setColor(0x00FFFF)
+        .setURL(url)
+        .setDescription(`ID: ${type}`);
+    if (thumbnail) embed.setThumbnail(thumbnail);
+    if (type === 'download_error') embed.setColor(0xFC2003);
+
+    const result = await api.webhooks.execute(webhook_id, webhook_token, {
+        content: body,
+        username: 'YoutubeDL-Material',
+        avatar_url: consts.ICON_URL,
+        embeds: [embed],
+    });
+    return result;
+}
+
+function sendGenericNotification(data) {
+    const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
+    logger.verbose(`Sending generic notification to ${webhook_url}`);
+    fetch(webhook_url, {
+        method: 'POST',
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify(data),
+    });
+}

backend/package.json

@@ -5,20 +5,9 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon app.js",
+    "start": "pm2-runtime --raw pm2.config.js",
     "debug": "set YTDL_MODE=debug && node app.js"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "*.js",
-      "appdata/*",
-      "public/*"
-    ],
-    "watch": [
-      "restart_update.json",
-      "restart_general.json"
-    ]
-  },
   "repository": {
     "type": "git",
     "url": ""
@@ -30,31 +19,36 @@
   },
   "homepage": "",
   "dependencies": {
+    "@discordjs/builders": "^1.6.1",
+    "@discordjs/core": "^0.5.2",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
-    "async-mutex": "^0.3.1",
+    "async-mutex": "^0.4.0",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
+    "express-session": "^1.17.3",
+    "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "gotify": "^1.1.0",
     "jsonwebtoken": "^8.5.1",
+    "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
-    "merge-files": "^0.1.2",
     "mocha": "^9.2.2",
-    "moment": "^2.29.2",
+    "moment": "^2.29.4",
     "mongodb": "^3.6.9",
-    "multer": "^1.4.2",
+    "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
-    "nodemon": "^2.0.7",
+    "node-telegram-bot-api": "^0.61.0",
     "passport": "^0.4.1",
     "passport-http": "^0.3.0",
-    "passport-jwt": "^4.0.0",
+    "passport-jwt": "^4.0.1",
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
@@ -63,7 +57,7 @@
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.0.6",
+    "uuidv4": "^6.2.13",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"

backend/pm2.config.js

@@ -6,4 +6,4 @@ module.exports = {
       out_file: "/dev/null",
       error_file: "/dev/null"
     }]
-}
+}

backend/subscriptions.js

@@ -3,6 +3,7 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
+const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
 
@@ -91,7 +92,10 @@ async function getSubscriptionInfo(sub) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
+                            let sub_name = sub.name;
+                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
+                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
                         }
                     }
 
@@ -138,28 +142,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
 
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     if (deleteMode && (await fs.pathExists(appendedBasePath))) {
-        if (sub.archive && (await fs.pathExists(sub.archive))) {
-            const archive_file_path = path.join(sub.archive, 'archive.txt');
-            // deletes archive if it exists
-            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
-            if (await fs.pathExists(archive_file_path)) {
-                await fs.unlink(archive_file_path);
-            }
-            await fs.rmdir(sub.archive);
-        }
         await fs.remove(appendedBasePath);
     }
+
+    await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
 async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    if (typeof sub === 'string') {
+        // TODO: fix bad workaround where sub is a sub_id
+        sub = await db_api.getRecord('subscriptions', {sub_id: sub});
+    }
     // TODO: combine this with deletefile
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
+    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -178,7 +179,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
+        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = info_json['id'];
+        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -196,12 +199,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (!deleteForever && useArchive && sub.archive && retrievedID) {
-                const archive_path = path.join(sub.archive, 'archive.txt')
-                // if archive exists, remove line with video ID
-                if (await fs.pathExists(archive_path)) {
-                    utils.removeIDFromArchive(archive_path, retrievedID);
-                }
+            const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useArchive && !deleteForever) {
+                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
         }
@@ -232,74 +232,39 @@ async function getVideosForSub(sub, user_uid = null) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable')) {
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        // TODO: reimplement
-
-                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
-                        // for (let i = 0; i < outputs.length; i++) {
-                        //     const output = JSON.parse(outputs[i]);
-                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                        //         // we found a video that errored! add it to the archive to prevent future errors
-                        //         if (sub.archive) {
-                        //             archive_dir = sub.archive;
-                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                        //             fs.appendFileSync(archive_path, output['id']);
-                        //         }
-                        //     }
-                        // }
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
+                        resolve(files_to_download);
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
                     }
+                } else {
+                    logger.error('Subscription check failed!');
                 }
                 resolve(false);
             } else if (output) {
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
-                    logger.verbose('No additional videos to download for ' + sub.name);
-                    resolve(true);
-                    return;
-                }
-
-                const output_jsons = [];
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                        output_jsons.push(output_json);
-                    } catch(e) {
-                        output_json = null;
-                    }
-                    if (!output_json) {
-                        continue;
-                    }
-                }
-
-                const files_to_download = await getFilesToDownload(sub, output_jsons);
-                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
-
-                for (let j = 0; j < files_to_download.length; j++) {
-                    const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
-                }
-
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
                 resolve(files_to_download);
            }
         });
@@ -309,6 +274,43 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+async function handleOutputJSON(output, sub, user_uid) {
+    if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+        await setFreshUploads(sub, user_uid);
+        checkVideosForFreshUploads(sub, user_uid);
+    }
+
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+        logger.verbose('No additional videos to download for ' + sub.name);
+        return [];
+    }
+
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
+    const files_to_download = await getFilesToDownload(sub, output_jsons);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+    for (let j = 0; j < files_to_download.length; j++) {
+        const file_to_download = files_to_download[j];
+        file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
+    }
+
+    return files_to_download;
+}
+
 function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
@@ -319,10 +321,10 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
 
     const base_download_options = {
-        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        maxHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
         customFileFolderPath: getAppendedBasePath(sub, basePath),
         customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        customArchivePath: path.join(basePath, 'archives', sub.name),
         additionalArgs: sub.custom_args
     }
 
@@ -337,8 +339,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -364,6 +364,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
+    // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
+        const archive_path = path.join(appendedBasePath, 'archive.txt');
+        await fs.writeFile(archive_path, archive_text);
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
@@ -374,26 +384,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
-    let archive_dir = null;
-    let archive_path = null;
-
-    if (useArchive && !redownload) {
-        if (sub.archive) {
-            archive_dir = sub.archive;
-            if (sub.type && sub.type === 'audio') {
-                archive_path = path.join(archive_dir, 'merged_audio.txt');
-            } else {
-                archive_path = path.join(archive_dir, 'merged_video.txt');
-            }            
-        }
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
-    // if streaming only mode, just get the list of videos
-    if (sub.streamingOnly) {
-        downloadConfig = ['-f', 'best', '--dump-json'];
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -418,9 +408,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
     if (default_downloader === 'yt-dlp') {
-        downloadConfig.push('--no-clean-infojson');
+        downloadConfig.push('--no-clean-info-json');
     }
 
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
     return downloadConfig;
 }
 
@@ -434,7 +426,14 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+                continue;
             }
+            const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useYoutubeDLArchive) {
+                const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+                if (exists_in_archive) continue;
+            }
+
             files_to_download.push(output_json);
         }
     }
@@ -453,7 +452,12 @@ async function getAllSubscriptions() {
 }
 
 async function getSubscription(subID) {
-    return await db_api.getRecord('subscriptions', {id: subID});
+    // stringify and parse because we may override the 'downloading' property
+    const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
+    // now with the download_queue, we may need to override 'downloading'
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
+    return sub;
 }
 
 async function getSubscriptionByName(subName, user_uid = null) {
@@ -467,7 +471,7 @@ async function updateSubscription(sub) {
 
 async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
-        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
+        await updateSubscriptionProperty(sub, assignment_obj);
     });
 }
 
@@ -479,6 +483,7 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
 
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    if (!sub_files) return;
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
     sub_files.forEach(async file => {
         if (current_date === file['upload_date'].replace(/-/g, '')) {

backend/tasks.js

@@ -1,5 +1,7 @@
 const db_api = require('./db');
+const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 const fs = require('fs-extra');
 const logger = require('./logger');
@@ -33,6 +35,28 @@ const TASKS = {
         confirm: youtubedl_api.updateYoutubeDL,
         title: 'Update youtube-dl',
         job: null
+    },
+    delete_old_files: {
+        run: checkForAutoDeleteFiles,
+        confirm: autoDeleteFiles,
+        title: 'Delete old files',
+        job: null
+    },
+    import_legacy_archives: {
+        run: archive_api.importArchives,
+        title: 'Import legacy archives',
+        job: null
+    }
+}
+
+const defaultOptions = {
+    all: {
+        auto_confirm: false
+    },
+    delete_old_files: {
+        blacklist_files: false,
+        blacklist_subscription_files: false,
+        threshold_days: ''
     }
 }
 
@@ -45,7 +69,7 @@ function scheduleJob(task_key, schedule) {
         const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
         const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
         const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
         return null;
@@ -57,7 +81,7 @@ function scheduleJob(task_key, schedule) {
             logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
             return;
         }
-        
+
         // remove schedule if it's a one-time task
         if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
         // we're just "running" the task, any confirmation should be user-initiated
@@ -77,9 +101,10 @@ exports.setupTasks = async () => {
     const tasks_keys = Object.keys(TASKS);
     for (let i = 0; i < tasks_keys.length; i++) {
         const task_key = tasks_keys[i];
+        const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
         const task_in_db = await db_api.getRecord('tasks', {key: task_key});
         if (!task_in_db) {
-            // insert task metadata into table if missing
+            // insert task metadata into table if missing, eventually move title to UI
             await db_api.insertRecordIntoTable('tasks', {
                 key: task_key,
                 title: TASKS[task_key]['title'],
@@ -90,9 +115,19 @@ exports.setupTasks = async () => {
                 data: null,
                 error: null,
                 schedule: null,
-                options: {}
+                options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
             });
         } else {
+            // verify all options exist in task
+            for (const key of Object.keys(mergedDefaultOptions)) {
+                const option_key = `options.${key}`;
+                // Remove any potential mangled option keys (#861)
+                await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
+                if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
+                    await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
+                }
+            }
+
             // reset task if necessary
             await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
 
@@ -123,15 +158,23 @@ exports.executeTask = async (task_key) => {
 
 exports.executeRun = async (task_key) => {
     logger.verbose(`Running task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     // don't set running to true when backup up DB as it will be stick "running" if restored
     if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
     const data = await TASKS[task_key].run();
     await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
     logger.verbose(`Finished running task ${task_key}`);
+    const task_obj = await db_api.getRecord('tasks', {key: task_key});
+    await notifications_api.sendTaskNotification(task_obj, false);
+
+    if (task_obj['options'] && task_obj['options']['auto_confirm']) {
+        exports.executeConfirm(task_key);
+    }
 }
 
 exports.executeConfirm = async (task_key) => {
     logger.verbose(`Confirming task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     if (!TASKS[task_key]['confirm']) {
         return null;
     }
@@ -141,6 +184,7 @@ exports.executeConfirm = async (task_key) => {
     await TASKS[task_key].confirm(data);
     await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
     logger.verbose(`Finished confirming task ${task_key}`);
+    await notifications_api.sendTaskNotification(task_obj, false);
 }
 
 exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -193,4 +237,31 @@ async function removeDuplicates(data) {
     }
 }
 
+// auto delete files
+
+async function checkForAutoDeleteFiles() {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
+        const error_message = 'Failed to do delete check because no limit was set!';
+        logger.error(error_message);
+        await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
+        return null;
+    }
+    const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
+    const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
+    const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
+    return {files_to_remove: files_to_remove};
+}
+
+async function autoDeleteFiles(data) {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (data['files_to_remove']) {
+        logger.info(`Removing ${data['files_to_remove'].length} old files!`);
+        for (let i = 0; i < data['files_to_remove'].length; i++) {
+            const file_to_remove = data['files_to_remove'][i];
+            await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
+        }
+    }
+}
+
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -1,8 +1,9 @@
-var assert = require('assert');
+/* eslint-disable no-undef */
+const assert = require('assert');
 const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
 
-process.chdir('./backend')
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -37,8 +38,11 @@ var auth_api = require('../authentication/auth');
 var db_api = require('../db');
 const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
+const archive_api = require('../archive');
+const categories_api = require('../categories');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
 
 db_api.initialize(db, users_db);
 
@@ -64,12 +68,12 @@ const sample_video_json = {
 
 describe('Database', async function() {
     describe('Import', async function() {
-        it('Migrate', async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords();
-            const success = await db_api.importJSONToDB(db.value(), users_db.value());
-            assert(success);
-        });
+        // it('Migrate', async function() {
+        //     await db_api.connectToDB();
+        //     await db_api.removeAllRecords();
+        //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
+        //     assert(success);
+        // });
 
         it('Transfer to remote', async function() {
             await db_api.removeAllRecords('test');
@@ -102,157 +106,208 @@ describe('Database', async function() {
         });
     });
 
-    describe('Export', function() {
-
-    });
-
-
     describe('Basic functions', async function() {
-        beforeEach(async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords('test');
-        });
-        it('Add and read record', async function() {
-            this.timeout(120000);
-            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
-            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
-            assert(added_record['test_add'] === 'test');
-            await db_api.removeRecord('test', {test_add: 'test'});
-        });
+        
+        // test both local_db and remote_db
+        const local_db_modes = [false, true];
 
-        it('Find duplicates by key', async function() {
-            const test_duplicates = [
-                {
-                    test: 'testing',
-                    key: '1'
-                },
-                {
-                    test: 'testing',
-                    key: '2'
-                },
-                {
-                    test: 'testing_missing',
-                    key: '3'
-                },
-                {
-                    test: 'testing',
-                    key: '4'
-                }
-            ];
-            await db_api.insertRecordsIntoTable('test', test_duplicates);
-            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-            console.log(duplicates);
-        });
-
-        it('Update record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
-            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
-            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
-            assert(updated_record['added_field']);
-            await db_api.removeRecord('test', {test_update: 'test'});
-        });
-
-        it('Remove record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
-            assert(delete_succeeded);
-            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
-            assert(!deleted_record);
-        });
-
-        it('Push to record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
-            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 1);
-        });
-
-        it('Pull from record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
-            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 0);
-        });
-
-        it('Bulk add', async function() {
-            this.timeout(120000);
-            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
-            const test_records = [];
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                test_records.push({
-                    uid: uuid()
+        for (const local_db_mode of local_db_modes) {
+            let use_local_db = local_db_mode;
+            describe(`Use local DB - ${use_local_db}`, async function() {
+                beforeEach(async function() {
+                    if (!use_local_db) {
+                        this.timeout(120000);
+                        await db_api.connectToDB(0);
+                    }
+                    await db_api.removeAllRecords('test');
                 });
-            }
-            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-
-            const received_records = await db_api.getRecords('test');
-            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
-        });
-
-        it('Bulk update', async function() {
-            // bulk add records
-            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
-            const test_records = [];
-            const update_obj = {};
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const test_uid =  uuid();
-                test_records.push({
-                    uid: test_uid
+                it('Add and read record', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+                    assert(added_record['test_add'] === 'test');
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Add and read record - Nested property', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
+                    const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
+                    assert(added_record['test_add'] === 'test');
+                    assert(!not_added_record);
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Replace filter', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
+                    assert(count === 1);
+                    await db_api.removeRecord('test', {test_replace_filter: 'test'});
+                });
+                it('Find duplicates by key', async function() {
+                    const test_duplicates = [
+                        {
+                            test: 'testing',
+                            key: '1'
+                        },
+                        {
+                            test: 'testing',
+                            key: '2'
+                        },
+                        {
+                            test: 'testing_missing',
+                            key: '3'
+                        },
+                        {
+                            test: 'testing',
+                            key: '4'
+                        }
+                    ];
+                    await db_api.insertRecordsIntoTable('test', test_duplicates);
+                    const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+                    console.log(duplicates);
                 });
-                update_obj[test_uid] = {added_field: true};
-            }
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            assert(success);
 
-            // makes sure they are added
-            const received_records = await db_api.getRecords('test');
-            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                it('Update record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+                    await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+                    const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+                    assert(updated_record['added_field']);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
-            assert(success);
+                it('Update records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
+                    await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
+                    const updated_records = await db_api.getRecords('test', {added_field: true});
+                    assert(updated_records.length === 2);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            const received_updated_records = await db_api.getRecords('test');
-            for (let i = 0; i < received_updated_records.length; i++) {
-                success &= received_updated_records[i]['added_field'];
-            }
-            assert(success);
-        });
+                it('Remove property from record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
+                    await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
+                    const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
+                    assert(updated_record['test_keep']);
+                    assert(!updated_record['test_remove']);
+                    await db_api.removeRecord('test', {test_keep: 'test'});
+                });
 
-        it('Stats', async function() {
-            const stats = await db_api.getDBStats();
-            assert(stats);
-        });
+                it('Remove record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+                    assert(!deleted_record);
+                });
 
-        it('Query speed', async function() {
-            this.timeout(120000); 
-            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
-            const test_records = [];
-            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const uid = uuid();
-                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
-            }
-            const insert_start = Date.now();
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            const insert_end = Date.now();
+                it('Remove records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
+                    assert(count === 0);
+                });
 
-            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+                it('Push to record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+                    await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 1);
+                });
 
-            const query_start = Date.now();
-            const random_record = await db_api.getRecord('test', {uid: random_uid});
-            const query_end = Date.now();
+                it('Pull from record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+                    await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 0);
+                });
 
-            console.log(random_record)
+                it('Bulk add', async function() {
+                    this.timeout(120000);
+                    const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+                    const test_records = [];
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        test_records.push({
+                            uid: uuid()
+                        });
+                    }
+                    const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
 
-            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+                    const received_records = await db_api.getRecords('test');
+                    assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                });
 
-            success = !!random_record;
+                it('Bulk update', async function() {
+                    // bulk add records
+                    const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+                    const test_records = [];
+                    const update_obj = {};
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const test_uid =  uuid();
+                        test_records.push({
+                            uid: test_uid
+                        });
+                        update_obj[test_uid] = {added_field: true};
+                    }
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    assert(success);
 
-            assert(success);
-        });
+                    // makes sure they are added
+                    const received_records = await db_api.getRecords('test');
+                    assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+                    success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
+                    assert(success);
+
+                    const received_updated_records = await db_api.getRecords('test');
+                    for (let i = 0; i < received_updated_records.length; i++) {
+                        success &= received_updated_records[i]['added_field'];
+                    }
+                    assert(success);
+                });
+
+                it('Stats', async function() {
+                    const stats = await db_api.getDBStats();
+                    assert(stats);
+                });
+
+                it('Query speed', async function() {
+                    this.timeout(120000); 
+                    const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+                    const test_records = [];
+                    let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const uid = uuid();
+                        if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                        test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                    }
+                    const insert_start = Date.now();
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    const insert_end = Date.now();
+
+                    console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+                    const query_start = Date.now();
+                    const random_record = await db_api.getRecord('test', {uid: random_uid});
+                    const query_end = Date.now();
+
+                    console.log(random_record)
+
+                    console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+                    success = !!random_record;
+
+                    assert(success);
+                });
+            });
+        }
     });
 
     describe('Local DB Filters', async function() {
@@ -281,24 +336,30 @@ describe('Database', async function() {
 });
 
 describe('Multi User', async function() {
-    let user = null;
-    const user_to_test = 'admin';
-    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
-    const playlist_to_test = 'ysabVZz4x';
+    const user_to_test = 'test_user';
+    const user_password = 'test_pass';
+    const sub_to_test = '';
+    const playlist_to_test = '';
     beforeEach(async function() {
         await db_api.connectToDB();
-        auth_api.initialize(db_api, logger);
-        subscriptions_api.initialize(db_api, logger);
-        user = await auth_api.login('admin', 'pass');
+        await auth_api.deleteUser(user_to_test);
     });
-    describe('Authentication', function() {
-        it('login', async function() {
+    describe('Basic', function() {
+        it('Register', async function() {
+            const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
+            assert(user);
+        });
+        it('Login', async function() {
+            await auth_api.registerUser(user_to_test, user_to_test, user_password);
+            const user = await auth_api.login(user_to_test, user_password);
             assert(user);
         });
     });
     describe('Video player - normal', async function() {
-        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
-        await db_api.insertRecordIntoTable('files', sample_video_json);
+        beforeEach(async function() {
+            await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+            await db_api.insertRecordIntoTable('files', sample_video_json);
+        });
         const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
             const video_obj = await db_api.getVideo(video_to_test);
@@ -306,14 +367,14 @@ describe('Multi User', async function() {
         });
 
         it('Video access - disallowed', async function() {
-            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
-            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(!video_obj);
         });
 
         it('Video access - allowed', async function() {
             await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
-            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(video_obj);
         });
     });
@@ -399,6 +460,19 @@ describe('Downloader', function() {
 
     });
 
+    it('Tag file', async function() {
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        const tags = {
+            title: sample_json['title'],
+            artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
+            TRCK: '27'
+        }
+        NodeID3.write(tags, audio_path);
+        const written_tags = NodeID3.read(audio_path);
+        assert(written_tags['raw']['TRCK'] === '27');
+    });
+
     it('Queue file', async function() {
         this.timeout(300000); 
         const returned_download = await downloader_api.createDownload(url, 'video', options);
@@ -442,14 +516,33 @@ describe('Downloader', function() {
         const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
         const updated_args1 = utils.injectArgs(original_args1, new_args1);
         const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
-        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
+        assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
 
         const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
         const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
         const updated_args2 = utils.injectArgs(original_args2, new_args2);
-        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
-        console.log(updated_args2);
-        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
+
+        const original_args3 = ['-o', '%(title)s.%(ext)s'];
+        const new_args3 =  ['--min-filesize','1'];
+        const updated_args3 = utils.injectArgs(original_args3, new_args3);
+        const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
+        assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
+    });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1710641401';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
     });
 });
 
@@ -521,7 +614,7 @@ describe('Tasks', function() {
         fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file, true);
+        assert(!!imported_file === true);
         
         // post-test cleanup
         if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -561,4 +654,207 @@ describe('Tasks', function() {
         const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
         assert(dummy_task_obj['data']);
     });
+});
+
+describe('Archive', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    it('Import archive', async function() {
+        const archive_text = `
+            testextractor1 testing1
+            testextractor1 testing2
+            testextractor2 testing1
+            testextractor1 testing3
+
+        `;
+        const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
+        assert(count === 4)
+        const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        console.log(archive_items);
+        assert(archive_items.length === 4);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
+
+        const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        assert(success);
+    });
+
+    it('Get archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+
+        assert(archive_item1 && archive_item2);
+    });
+
+    it('Archive duplicates', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
+
+        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
+        assert(count === 3);
+    });
+
+    it('Remove from archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
+
+        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
+        assert(success);
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        assert(!!archive_item1);
+        
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+        assert(!archive_item2);
+
+        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
+        assert(!!archive_item3);
+    });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
+    });
+
+    it('Convert flat object to nested object', async function() {
+        // No modfication
+        const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
+        const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
+        assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
+        assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
+
+        // Standard setup
+        const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
+        const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
+        assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
+        assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
+
+        // Nested branches
+        const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
+        const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
+    });
+});
+
+describe('Categories', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        const new_category = {
+            name: 'test_category',
+            uid: uuid(),
+            rules: [],
+            custom_output: ''
+        };
+    
+        await db_api.insertRecordIntoTable('categories', new_category);
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('categories', {name: 'test_category'});
+    });
+
+    it('Categorize - includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        console.log(category);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - AND', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'and',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - OR', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'or',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category);
+    });
 });

backend/twitch.js

@@ -1,90 +1,73 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
 const config_api = require('./config');
+const logger = require('./logger');
 
-async function getCommentsForVOD(clientID, vodId) {
-    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
-        batch,
-        cursor;
+const moment = require('moment');
+const fs = require('fs-extra')
+const path = require('path');
+const { promisify } = require('util');
+const child_process = require('child_process');
 
-    let comments = null;
-
-    try {
-        do {
-            batch = (await Axios.get(url, {
-                headers: {
-                    'Client-ID': clientID,
-                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
-                    'Content-Type': 'application/json; charset=UTF-8',
-                }
-            })).data;
-
-            const str = batch.comments.map(c => {
-                let {
-                        created_at: msgCreated,
-                        content_offset_seconds: timestamp,
-                        commenter: {
-                            name,
-                            _id,
-                            created_at: acctCreated
-                        },
-                        message: {
-                            body: msg,
-                            user_color: user_color
-                        }
-                    } = c;
-
-                const timestamp_str = moment.duration(timestamp, 'seconds')
-                    .toISOString()
-                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
-                        (_, ...ms) => {
-                            const seg = v => v ? v.padStart(2, '0') : '00';
-                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
-                        });
-
-                acctCreated = moment(acctCreated).utc();
-                msgCreated = moment(msgCreated).utc();
-
-                if (!comments) comments = [];
-
-                comments.push({
-                    timestamp: timestamp,
-                    timestamp_str: timestamp_str,
-                    name: name,
-                    message: msg,
-                    user_color: user_color
-                });
-                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
-                // return line;
-            }).join('\n');
-
-            cursor = batch._next;
-            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
-            await new Promise(res => setTimeout(res, 300));
-        } while (cursor);
-    } catch (err) {
-        console.error(err);
+async function getCommentsForVOD(vodId) {
+    const exec = promisify(child_process.exec);
+    
+    // Reject invalid params to prevent command injection attack
+    if (!vodId.match(/^[0-9a-z]+$/)) {
+        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
+        return null;
     }
 
-    return comments;
+    const is_windows = process.platform === 'win32';
+    const cliExt = is_windows ? '.exe' : ''
+    const cliPath = `TwitchDownloaderCLI${cliExt}`
+
+    if (!fs.existsSync(cliPath)) {
+        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
+        return null;
+    }
+
+    const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
+
+    if (result['stderr']) {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
+        logger.error(result['stderr']);
+        return null;
+    }
+
+    const temp_chat_path = path.join('appdata', `${vodId}.json`);
+
+    const raw_json = fs.readJSONSync(temp_chat_path);
+    const new_json = raw_json.comments.map(comment_obj => {
+        return {
+            timestamp: comment_obj.content_offset_seconds,
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+            name: comment_obj.commenter.name,
+            message: comment_obj.message.body,
+            user_color: comment_obj.message.user_color
+        }
+    });
+
+    fs.unlinkSync(temp_chat_path);
+
+    return new_json;
 }
 
 async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
     let file_path = null;
 
     if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
         }
     }
 
@@ -96,23 +79,26 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
     return chat_file;
 }
 
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
-    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
-    const chat = await getCommentsForVOD(twitch_api_key, vodId);
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const chat = await getCommentsForVOD(vodId);
 
     // save file if needed params are included
     let file_path = null;
-    if (user_uid) {
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            file_path = path.join(type, `${id}.twitch_chat.json`);
         }
     }
 
@@ -121,6 +107,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
     return chat;
 }
 
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
 module.exports = {
     getCommentsForVOD: getCommentsForVOD,
     getTwitchChatByFileID: getTwitchChatByFileID,

backend/utils.js

@@ -4,6 +4,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const archiver = require('archiver');
 const fetch = require('node-fetch');
 const ProgressBar = require('progress');
+const winston = require('winston');
 
 const config_api = require('./config');
 const logger = require('./logger');
@@ -12,7 +13,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-function getTrueFileName(unfixed_path, type) {
+exports.getTrueFileName = (unfixed_path, type) => {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -27,13 +28,13 @@ function getTrueFileName(unfixed_path, type) {
     return fixed_path;
 }
 
-async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
+exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
     // return empty array if the path doesn't exist
     if (!(await fs.pathExists(basePath))) return [];
 
     let files = [];
     const ext = type === 'audio' ? 'mp3' : 'mp4';
-    var located_files = await recFindByExt(basePath, ext);
+    var located_files = await exports.recFindByExt(basePath, ext);
     for (let i = 0; i < located_files.length; i++) {
         let file = located_files[i];
         var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -41,33 +42,33 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
         var stats = await fs.stat(file);
 
         var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await getJSONByType(type, id, basePath);
+        var jsonobj = await exports.getJSONByType(type, id, basePath);
         if (!jsonobj) continue;
         if (full_metadata) {
             jsonobj['id'] = id;
             files.push(jsonobj);
             continue;
         }
-        var upload_date = formatDateString(jsonobj.upload_date);
+        var upload_date = exports.formatDateString(jsonobj.upload_date);
 
         var isaudio = type === 'audio';
-        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+        var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
                                 stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
 }
 
-async function createContainerZipFile(file_name, container_file_objs) {
+exports.createContainerZipFile = async (file_name, container_file_objs) => {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
 }
 
-async function createZipFile(zip_file_path, file_paths) {
+exports.createZipFile = async (zip_file_path, file_paths) => {
     let output = fs.createWriteStream(zip_file_path);
 
     var archive = archiver('zip', {
@@ -91,11 +92,11 @@ async function createZipFile(zip_file_path, file_paths) {
     await archive.finalize();
 
     // wait a tiny bit for the zip to reload in fs
-    await wait(100);
+    await exports.wait(100);
     return zip_file_path;
 }
 
-function getJSONMp4(name, customPath, openReadPerms = false) {
+exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -110,7 +111,7 @@ function getJSONMp4(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSONMp3(name, customPath, openReadPerms = false) {
+exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
     var obj = null;
     if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -127,11 +128,11 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSON(file_path, type) {
+exports.getJSON = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
     let obj = null;
-    var jsonPath = removeFileExtension(file_path) + '.info.json';
-    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
+    var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
     if (fs.existsSync(jsonPath))
     {
         obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -142,12 +143,12 @@ function getJSON(file_path, type) {
     return obj;
 }
 
-function getJSONByType(type, name, customPath, openReadPerms = false) {
-    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
+    return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(file_path) {
-    const file_path_no_extension = removeFileExtension(file_path);
+exports.getDownloadedThumbnail = (file_path) => {
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
     let webpPath = file_path_no_extension + '.webp';
@@ -163,7 +164,7 @@ function getDownloadedThumbnail(file_path) {
         return null;
 }
 
-function getExpectedFileSize(input_info_jsons) {
+exports.getExpectedFileSize = (input_info_jsons) => {
     // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
     const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
 
@@ -172,11 +173,13 @@ function getExpectedFileSize(input_info_jsons) {
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
-            info_json.formats.forEach(available_format => {
-                if (available_format.format_id === format_id && available_format.filesize) {
-                    individual_expected_filesize += available_format.filesize;
-                }
-            });
+            if (info_json.formats !== undefined) {
+                info_json.formats.forEach(available_format => {
+                  if (available_format.format_id === format_id && (available_format.filesize || available_format.filesize_approx)) {
+                    individual_expected_filesize += (available_format.filesize ? available_format.filesize : available_format.filesize_approx);
+                  }
+                });
+            }
         });
         expected_filesize += individual_expected_filesize;
     });
@@ -184,12 +187,12 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(file_path, type) {
+exports.fixVideoMetadataPerms = (file_path, type) => {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     const files_to_fix = [
         // JSONs
@@ -206,11 +209,11 @@ function fixVideoMetadataPerms(file_path, type) {
     }
 }
 
-function deleteJSONFile(file_path, type) {
+exports.deleteJSONFile = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
-    
+    const file_path_no_extension = exports.removeFileExtension(file_path);
+
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
 
@@ -218,33 +221,7 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-async function removeIDFromArchive(archive_path, id) {
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
-    if (!data) {
-        logger.error('Archive could not be found.');
-        return;
-    }
-
-    let dataArray = data.split('\n'); // convert file data in an array
-    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-    let lastIndex = -1; // let say, we have not found the keyword
-
-    for (let index=0; index<dataArray.length; index++) {
-        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-            lastIndex = index; // found a line includes a id keyword
-            break;
-        }
-    }
-
-    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-
-    // UPDATE FILE WITH NEW DATA
-    const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
-    if (line) return line;
-}
-
-function durationStringToNumber(dur_str) {
+exports.durationStringToNumber = (dur_str) => {
     if (typeof dur_str === 'number') return dur_str;
     let num_sum = 0;
     const dur_str_parts = dur_str.split(':');
@@ -254,23 +231,22 @@ function durationStringToNumber(dur_str) {
     return num_sum;
 }
 
-function getMatchingCategoryFiles(category, files) {
+exports.getMatchingCategoryFiles = (category, files) => {
     return files && files.filter(file => file.category && file.category.uid === category.uid);
 }
 
-function addUIDsToCategory(category, files) {
-    const files_that_match = getMatchingCategoryFiles(category, files);
+exports.addUIDsToCategory = (category, files) => {
+    const files_that_match = exports.getMatchingCategoryFiles(category, files);
     category['uids'] = files_that_match.map(file => file.uid);
     return files_that_match;
 }
 
-function getCurrentDownloader() {
+exports.getCurrentDownloader = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     return details_json['downloader'];
 }
 
-async function recFindByExt(base, ext, files, result, recursive = true)
-{
+exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
     files = files || (await fs.readdir(base))
     result = result || []
 
@@ -279,7 +255,7 @@ async function recFindByExt(base, ext, files, result, recursive = true)
         if ( (await fs.stat(newbase)).isDirectory() )
         {
             if (!recursive) continue;
-            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+            result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
         {
@@ -292,23 +268,23 @@ async function recFindByExt(base, ext, files, result, recursive = true)
     return result
 }
 
-function removeFileExtension(filename) {
+exports.removeFileExtension = (filename) => {
     const filename_parts = filename.split('.');
     filename_parts.splice(filename_parts.length - 1);
     return filename_parts.join('.');
 }
 
-function formatDateString(date_string) {
+exports.formatDateString = (date_string) => {
     return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
 }
 
-function createEdgeNGrams(str) {
+exports.createEdgeNGrams = (str) => {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
-        
+
         return str.split(" ").reduce((ngrams, token) => {
-            if (token.length > minGram) {   
+            if (token.length > minGram) {
                 for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
                     ngrams = [...ngrams, token.substr(0, i)]
                 }
@@ -318,13 +294,13 @@ function createEdgeNGrams(str) {
             return ngrams
         }, []).join(" ")
     }
-    
+
     return str
 }
 
 // ffmpeg helper functions
 
-async function cropFile(file_path, start, end, ext) {
+exports.cropFile = async (file_path, start, end, ext) => {
     return new Promise(resolve => {
         const temp_file_path = `${file_path}.cropped${ext}`;
         let base_ffmpeg_call = ffmpeg(file_path);
@@ -353,13 +329,13 @@ async function cropFile(file_path, start, end, ext) {
  * setTimeout, but its a promise.
  * @param {number} ms
  */
- async function wait(ms) {
+exports.wait = async (ms) => {
     await new Promise(resolve => {
         setTimeout(resolve, ms);
     });
 }
 
-async function checkExistsWithTimeout(filePath, timeout) {
+exports.checkExistsWithTimeout = async (filePath, timeout) => {
     return new Promise(function (resolve, reject) {
 
         var timer = setTimeout(function () {
@@ -388,7 +364,7 @@ async function checkExistsWithTimeout(filePath, timeout) {
 }
 
 // helper function to download file using fetch
-async function fetchFile(url, path, file_label) {
+exports.fetchFile = async (url, path, file_label) => {
     var len = null;
     const res = await fetch(url);
 
@@ -415,10 +391,10 @@ async function fetchFile(url, path, file_label) {
     });
 }
 
-async function restartServer(is_update = false) {
+exports.restartServer = async (is_update = false) => {
     logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
 
-    // the following line restarts the server through nodemon
+    // the following line restarts the server through pm2
     fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
     process.exit(1);
 }
@@ -428,20 +404,21 @@ async function restartServer(is_update = false) {
 //  - if already exists and doesn't have value, ignore
 //  - if it doesn't exist and has value, add both arg and value
 //  - if it doesn't exist and doesn't have value, add arg
-function injectArgs(original_args, new_args) {
+exports.injectArgs = (original_args, new_args) => {
     const updated_args = original_args.slice();
     try {
         for (let i = 0; i < new_args.length; i++) {
             const new_arg = new_args[i];
             if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
-            
+
             if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
                 if (original_args.includes(new_arg)) {
                     const original_index = original_args.indexOf(new_arg);
-                    original_args.splice(original_index, 2);
+                    updated_args.splice(original_index, 2);
                 }
 
                 updated_args.push(new_arg, new_args[i + 1]);
+                i++; // we need to skip the arg value on the next loop
             } else {
                 if (!original_args.includes(new_arg)) {
                     updated_args.push(new_arg);
@@ -456,7 +433,11 @@ function injectArgs(original_args, new_args) {
     return updated_args;
 }
 
-const searchObjectByString = function(o, s) {
+exports.filterArgs = (args, args_to_remove) => {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+exports.searchObjectByString = (o, s) => {
     s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
     s = s.replace(/^\./, '');           // strip a leading dot
     var a = s.split('.');
@@ -471,6 +452,73 @@ const searchObjectByString = function(o, s) {
     return o;
 }
 
+exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
+exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
+exports.getBaseURL = () => {
+    return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
+}
+
+exports.updateLoggerLevel = (new_logger_level) => {
+    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    if (!possible_levels.includes(new_logger_level)) {
+        logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
+        new_logger_level = 'info';
+    }
+    logger.level = new_logger_level;
+    winston.loggers.get('console').level = new_logger_level;
+    logger.transports[2].level = new_logger_level;
+}
+
+exports.convertFlatObjectToNestedObject = (obj) => {
+    const result = {};
+    for (const key in obj) {
+      const nestedKeys = key.split('.');
+      let currentObj = result;
+      for (let i = 0; i < nestedKeys.length; i++) {
+        if (i === nestedKeys.length - 1) {
+          currentObj[nestedKeys[i]] = obj[key];
+        } else {
+          currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
+          currentObj = currentObj[nestedKeys[i]];
+        }
+      }
+    }
+    return result;
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -488,33 +536,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.view_count = view_count;
     this.height = height;
     this.abr = abr;
-}
+    this.favorite = false;
+}   
+exports.File = File;
 
-module.exports = {
-    getJSONMp3: getJSONMp3,
-    getJSONMp4: getJSONMp4,
-    getJSON: getJSON,
-    getTrueFileName: getTrueFileName,
-    getDownloadedThumbnail: getDownloadedThumbnail,
-    getExpectedFileSize: getExpectedFileSize,
-    fixVideoMetadataPerms: fixVideoMetadataPerms,
-    deleteJSONFile: deleteJSONFile,
-    removeIDFromArchive: removeIDFromArchive,
-    getDownloadedFilesByType: getDownloadedFilesByType,
-    createContainerZipFile: createContainerZipFile,
-    durationStringToNumber: durationStringToNumber,
-    getMatchingCategoryFiles: getMatchingCategoryFiles,
-    getCurrentDownloader: getCurrentDownloader,
-    recFindByExt: recFindByExt,
-    removeFileExtension: removeFileExtension,
-    formatDateString: formatDateString,
-    cropFile: cropFile,
-    createEdgeNGrams: createEdgeNGrams,
-    wait: wait,
-    checkExistsWithTimeout: checkExistsWithTimeout,
-    fetchFile: fetchFile,
-    restartServer: restartServer,
-    injectArgs: injectArgs,
-    searchObjectByString: searchObjectByString,
-    File: File
-}

backend/youtube-dl.js

@@ -90,7 +90,7 @@ exports.updateYoutubeDL = async (latest_update_version) => {
 
 exports.verifyBinaryExistsLinux = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
-    if (!is_windows && details_json && (details_json['path'].includes('.exe') || !details_json['path'])) {
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
         details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
         details_json['exec'] = 'youtube-dl';
         details_json['version'] = OUTDATED_VERSION;

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.2"
+appVersion: "4.3.1"

docker-compose.yml

@@ -2,7 +2,6 @@ version: "2"
 services:
     ytdl_material:
         environment: 
-            ALLOW_CONFIG_MUTATIONS: 'true'
             ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
             ytdl_use_local_db: 'false'
             write_ytdl_config: 'true'
@@ -17,14 +16,12 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:nightly
+        image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        image: mongo
-        ports:
-            - "27017:27017"
+        image: mongo:4
         logging:
             driver: "none"          
         container_name: mongo-db
         restart: always
         volumes:
-            - ./db/:/data/db
+            - ./db/:/data/db

docker-utils/GetTwitchDownloader.py

@@ -0,0 +1,53 @@
+import platform
+import requests
+import shutil
+import os
+import re
+
+from github import Github
+
+machine = platform.machine()
+
+def isARM():
+    return True if machine.startswith('arm') else False
+
+def getLatestFileInRepo(repo, search_string):
+    # Create an unauthenticated instance of the Github object
+    g = Github(os.environ.get('GH_TOKEN'))
+
+    # Replace with the repository owner and name
+    repo = g.get_repo(repo)
+
+    # Get all releases of the repository
+    releases = repo.get_releases()
+
+    # Loop through the releases in reverse order (from latest to oldest)
+    for release in list(releases):
+        # Get the release assets (files attached to the release)
+        assets = release.get_assets()
+
+        # Loop through the assets
+        for asset in assets:
+            if re.search(search_string, asset.name):
+                print(f'Downloading: {asset.name}')
+                response = requests.get(asset.browser_download_url)
+                with open(asset.name, 'wb') as f:
+                    f.write(response.content)
+                print(f'Download complete: {asset.name}. Unzipping...')
+                shutil.unpack_archive(asset.name, './')
+                print(f'Unzipping complete!')
+                os.remove(asset.name)
+                break
+        else:
+            continue
+        break
+    else:
+        # If no matching release is found, print a message
+        print(f'No release found with {search_string}')
+
+def getLatestCLIRelease():
+    isArm = isARM()
+    searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
+    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
+
+getLatestCLIRelease()

docker-utils/ffmpeg-fetch.sh

@@ -26,7 +26,7 @@ apt-get update && apt-get -y install curl xz-utils
 echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
 curl -o ffmpeg.txz \
     --connect-timeout 5 \
-    --max-time 10 \
+    --max-time 120 \
     --retry 5 \
     --retry-delay 0 \
     --retry-max-time 40 \

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.2.0",
+  "version": "4.3.1",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -13,7 +13,7 @@
     "e2e": "ng e2e",
     "electron": "ng build --base-href ./ && electron .",
     "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
-    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n"
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n --out-file=messages.en.xlf"
   },
   "engines": {
     "node": "12.3.1",
@@ -21,62 +21,62 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
-    "@angular/animations": "^13.3.4",
-    "@angular/cdk": "^13.3.4",
-    "@angular/common": "^13.3.4",
-    "@angular/compiler": "^13.3.4",
-    "@angular/core": "^13.3.4",
-    "@angular/forms": "^13.3.4",
-    "@angular/localize": "^13.3.4",
-    "@angular/material": "^13.3.4",
-    "@angular/platform-browser": "^13.3.4",
-    "@angular/platform-browser-dynamic": "^13.3.4",
-    "@angular/router": "^13.3.4",
+    "@angular-devkit/core": "^15.0.1",
+    "@angular/animations": "^15.0.1",
+    "@angular/cdk": "^15.0.0",
+    "@angular/common": "^15.0.1",
+    "@angular/compiler": "^15.0.1",
+    "@angular/core": "^15.0.1",
+    "@angular/forms": "^15.0.1",
+    "@angular/localize": "^15.0.1",
+    "@angular/material": "^15.0.0",
+    "@angular/platform-browser": "^15.0.1",
+    "@angular/platform-browser-dynamic": "^15.0.1",
+    "@angular/router": "^15.0.1",
     "@fontsource/material-icons": "^4.5.4",
     "@ngneat/content-loader": "^5.0.0",
-    "@videogular/ngx-videogular": "^5.0.1",
+    "@videogular/ngx-videogular": "^6.0.0",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
-    "filesize": "^6.1.0",
+    "filesize": "^10.0.7",
     "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
-    "ng-lazyload-image": "^7.0.1",
-    "ngx-avatars": "^1.3.1",
+    "ngx-avatars": "^1.4.1",
     "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
-    "rxjs-compat": "^6.0.0-rc.0",
+    "rxjs-compat": "^6.6.7",
     "tslib": "^2.0.0",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "xliff-to-json": "^1.0.4",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
-    "@angular/cli": "^13.3.3",
-    "@angular/compiler-cli": "^13.3.4",
-    "@angular/language-service": "^13.3.4",
+    "@angular-devkit/build-angular": "^15.0.1",
+    "@angular/cli": "^15.0.1",
+    "@angular/compiler-cli": "^15.0.1",
+    "@angular/language-service": "^15.0.1",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "~3.6.0",
+    "@types/jasmine": "^4.3.1",
     "@types/node": "^12.11.1",
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
+    "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
-    "electron": "^13.6.6",
+    "electron": "^19.1.9",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.3.16",
+    "karma": "~6.4.2",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
-    "karma-jasmine": "~4.0.0",
+    "karma-jasmine": "~5.1.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.21.0",
+    "openapi-typescript-codegen": "^0.23.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
 export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
@@ -26,8 +27,10 @@ export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
 export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
 export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
 export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -40,14 +43,18 @@ export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchCh
 export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
 export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
 export { FileType } from './models/FileType';
+export { FileTypeFilter } from './models/FileTypeFilter';
 export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
 export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
 export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
 export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
 export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetArchivesRequest } from './models/GetArchivesRequest';
+export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -61,6 +68,7 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
 export type { GetLogsResponse } from './models/GetLogsResponse';
 export type { GetMp3sResponse } from './models/GetMp3sResponse';
 export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
 export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
 export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
 export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -71,17 +79,24 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
 export type { LoginResponse } from './models/LoginResponse';
+export type { Notification } from './models/Notification';
+export { NotificationAction } from './models/NotificationAction';
+export { NotificationType } from './models/NotificationType';
 export type { Playlist } from './models/Playlist';
 export type { RegisterRequest } from './models/RegisterRequest';
 export type { RegisterResponse } from './models/RegisterResponse';
+export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
 export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
 export type { SharingToggle } from './models/SharingToggle';
+export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
 export type { SubscribeResponse } from './models/SubscribeResponse';
 export type { Subscription } from './models/Subscription';
@@ -105,8 +120,10 @@ export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';
 export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
+export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/Archive.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type Archive = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    title: string;
+    user_uid?: string;
+    sub_id?: string;
+    timestamp: number;
+    uid: string;
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -2,11 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
-
 export type CreatePlaylistRequest = {
     playlistName: string;
     uids: Array<string>;
-    type: FileType;
     thumbnailURL: string;
 };

src/api-types/models/DBInfoResponse.ts

@@ -14,5 +14,6 @@ subscriptions?: TableInfo;
 users?: TableInfo;
 roles?: TableInfo;
 download_queue?: TableInfo;
+archives?: TableInfo;
 };
 };

src/api-types/models/DatabaseFile.ts

@@ -26,8 +26,20 @@ export type DatabaseFile = {
     path: string;
     upload_date: string;
     uid: string;
+    user_uid?: string;
     sharingEnabled?: boolean;
     category?: Category;
     view_count?: number;
     local_view_count?: number;
+    sub_id?: string;
+    registered?: number;
+    /**
+     * In pixels, only for videos
+     */
+    height?: number;
+    /**
+     * In Kbps
+     */
+    abr?: number;
+    favorite: boolean;
 };

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type DeleteArchiveItemsRequest = {
+    archives: Array<Archive>;
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteNotificationRequest = {
+    uid: string;
+};

src/api-types/models/DeletePlaylistRequest.ts

@@ -2,9 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
-
 export type DeletePlaylistRequest = {
     playlist_id: string;
-    type: FileType;
 };

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,12 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { SubscriptionRequestData } from './SubscriptionRequestData';
-
 export type DeleteSubscriptionFileRequest = {
-    file: string;
-    file_uid?: string;
-    sub: SubscriptionRequestData;
+    file_uid: string;
     /**
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */

src/api-types/models/Download.ts

@@ -19,7 +19,12 @@ export type Download = {
      * Error text, set if download fails.
      */
     error?: string | null;
+    /**
+     * Error type, may or may not be set in case of an error
+     */
+    error_type?: string | null;
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
+    prefetched_info?: any;
 };

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,8 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { FileType } from './FileType';
+
 export type DownloadArchiveRequest = {
-    sub: {
-archive_dir: string;
-};
+    type?: FileType;
+    sub_id?: string;
 };

src/api-types/models/DownloadRequest.ts

@@ -35,10 +35,18 @@ export type DownloadRequest = {
      * Height of the video, if known
      */
     selectedHeight?: string;
+    /**
+     * Max height that should be used, useful for playlists. selectedHeight will override this.
+     */
+    maxHeight?: string;
     /**
      * Specify ffmpeg/avconv audio quality
      */
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
+    /**
+     * If using youtube-dl archive, download will ignore it
+     */
+    ignoreArchive?: boolean;
 };

src/api-types/models/FileTypeFilter.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum FileTypeFilter {
+    AUDIO_ONLY = 'audio_only',
+    VIDEO_ONLY = 'video_only',
+    BOTH = 'both',
+}

src/api-types/models/GetAllFilesRequest.ts

@@ -0,0 +1,24 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileTypeFilter } from './FileTypeFilter';
+import type { Sort } from './Sort';
+
+export type GetAllFilesRequest = {
+    sort?: Sort;
+    range?: Array<number>;
+    /**
+     * Filter files by title
+     */
+    text_search?: string;
+    file_type_filter?: FileTypeFilter;
+    /**
+     * If set to true, only gets favorites
+     */
+    favorite_filter?: boolean;
+    /**
+     * Include if you want to filter by subscription
+     */
+    sub_id?: string;
+};

src/api-types/models/GetArchivesRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type GetArchivesRequest = {
+    type?: FileType;
+    sub_id?: string;
+};

src/api-types/models/GetArchivesResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type GetArchivesResponse = {
+    archives: Array<Archive>;
+};

src/api-types/models/GetNotificationsResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Notification } from './Notification';
+
+export type GetNotificationsResponse = {
+    notifications?: Array<Notification>;
+};

src/api-types/models/GetPlaylistResponse.ts

@@ -2,11 +2,14 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
+import type { DatabaseFile } from './DatabaseFile';
 import type { Playlist } from './Playlist';
 
 export type GetPlaylistResponse = {
     playlist: Playlist;
-    type: FileType;
     success: boolean;
+    /**
+     * File objects for every uid in the playlist's uids property, in the same order
+     */
+    file_objs?: Array<DatabaseFile>;
 };

src/api-types/models/ImportArchiveRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type ImportArchiveRequest = {
+    archive: string;
+    type: FileType;
+    sub_id?: string;
+};

src/api-types/models/Notification.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { NotificationAction } from './NotificationAction';
+import type { NotificationType } from './NotificationType';
+
+export type Notification = {
+    type: NotificationType;
+    uid: string;
+    user_uid?: string;
+    action?: Array<NotificationAction>;
+    read: boolean;
+    data?: any;
+    timestamp: number;
+};

src/api-types/models/NotificationAction.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationAction {
+    PLAY = 'play',
+    RETRY_DOWNLOAD = 'retry_download',
+    VIEW_DOWNLOAD_ERROR = 'view_download_error',
+    VIEW_TASKS = 'view_tasks',
+}

src/api-types/models/NotificationType.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationType {
+    DOWNLOAD_COMPLETE = 'download_complete',
+    DOWNLOAD_ERROR = 'download_error',
+    TASK_FINISHED = 'task_finished',
+}

src/api-types/models/Playlist.ts

@@ -14,4 +14,5 @@ export type Playlist = {
     duration: number;
     user_uid?: string;
     auto?: boolean;
+    sharingEnabled?: boolean;
 };

src/api-types/models/RestartDownloadResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { SuccessObject } from './SuccessObject';
+
+export type RestartDownloadResponse = (SuccessObject & {
+new_download_uid?: string;
+});

src/api-types/models/Schedule.ts

@@ -9,6 +9,7 @@ dayOfWeek?: Array<number>;
 hour?: number;
 minute?: number;
 timestamp?: number;
+tz?: string;
 };
 };
 

src/api-types/models/SetNotificationsToReadRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type SetNotificationsToReadRequest = {
+    uids: Array<string>;
+};

src/api-types/models/Sort.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type Sort = {
+    /**
+     * Property to sort by
+     */
+    by?: string;
+    /**
+     * 1 for ascending, -1 for descending
+     */
+    order?: number;
+};

src/api-types/models/Subscription.ts

@@ -10,7 +10,6 @@ export type Subscription = {
     id: string;
     type: FileType;
     user_uid: string | null;
-    streamingOnly: boolean;
     isPlaylist: boolean;
     archive?: string;
     timerange?: string;

src/api-types/models/Task.ts

@@ -4,6 +4,7 @@
 
 export type Task = {
     key: string;
+    title?: string;
     last_ran: number;
     last_confirmed: number;
     running: boolean;
@@ -11,4 +12,5 @@ export type Task = {
     data: any;
     error: string;
     schedule: any;
+    options?: any;
 };

src/api-types/models/UpdateTaskOptionsRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UpdateTaskOptionsRequest = {
+    task_key: string;
+    new_options: any;
+};

src/api-types/models/UploadCookiesRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UploadCookiesRequest = {
+    cookies: Blob;
+};

src/app/app-routing.module.ts

@@ -23,7 +23,7 @@ const routes: Routes = [
 ];
 
 @NgModule({
-  imports: [RouterModule.forRoot(routes, { useHash: true, relativeLinkResolution: 'legacy' })],
+  imports: [RouterModule.forRoot(routes, { useHash: true })],
   exports: [RouterModule]
 })
 export class AppRoutingModule { }

src/app/app.component.css

@@ -13,7 +13,6 @@
 }
 
 .theme-slide-toggle {
-    top: 2px;
     left: 10px;
     position: relative;
     pointer-events: none;
@@ -25,4 +24,20 @@
 
 .top-toolbar {
     height: 64px;
-}
+    background: unset;
+}
+
+::ng-deep .top-menu-button > span {
+    width: 85px;
+    height: 26px;
+}
+
+::ng-deep .mdc-switch {
+    outline: none !important;
+}
+
+::ng-deep .notifications-menu {
+    width: 30vw !important;
+    max-width: 100% !important;
+    min-width: 280px !important;
+}

src/app/app.component.html

@@ -1,37 +1,41 @@
 <div [style.background]="postsService.theme ? postsService.theme.background_color : null" style="width: 100%; height: 100%;">
     <div class="mat-elevation-z3" style="position: relative; z-index: 10;">
-        <mat-toolbar color="primary" class="sticky-toolbar top-toolbar">
-            <div class="flex-row" width="100%" height="100%">
-                <div class="flex-column" style="text-align: left; margin-top: 1px;">
-                    <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
-                    <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
-                </div>
-                <div class="flex-column" style="text-align: center; margin-top: 5px;">
-                    <div style="font-size: 22px; text-shadow: #141414 0.25px 0.25px 1px;">
-                        {{topBarTitle}}
+        <mat-toolbar class="sticky-toolbar top-toolbar">
+            <div class="container-fluid" style="padding-left: 0px; padding-right: 0px">
+                <div class="row" width="100%" height="100%">
+                    <div class="col-6" style="text-align: left; margin-top: 1px;">
+                        <div style="display: flex; align-items: center;">
+                            <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
+                            <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
+                            <div style="margin-left: 8px; display: inline-block;"><button mat-icon-button routerLink='/home'><img style="width: 32px;" src="assets/images/logo_128px.png"></button></div>
+                        </div>
+                    </div>
+                    <div class="col-6" style="text-align: right; align-items: flex-end; display: inline-block">
+                        <button *ngIf="postsService.config?.Extra.enable_notifications" [matMenuTriggerFor]="notificationsMenu" (menuOpened)="notificationMenuOpened()" mat-icon-button><mat-icon [matBadge]="notification_count" matBadgeColor="warn" matBadgeSize="small" *ngIf="notification_count > 0">notifications</mat-icon><mat-icon *ngIf="notification_count === 0">notifications_none</mat-icon></button>
+                        <mat-menu [classList]="'notifications-menu'" (close)="notificationMenuClosed()" #notificationsMenu="matMenu">
+                            <app-notifications #notifications (notificationCount)="notificationCountUpdate($event)" (click)="$event.stopPropagation()"></app-notifications>
+                        </mat-menu>
+                        <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
+                        <mat-menu #menuSettings="matMenu">
+                            <button class="top-menu-button" (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
+                                <mat-icon>person</mat-icon>
+                                <span i18n="Profile menu label">Profile</span>
+                            </button>
+                            <button *ngIf="postsService.config && postsService.config.Downloader.use_youtubedl_archive" class="top-menu-button" (click)="openArchivesDialog()" mat-menu-item>
+                                <mat-icon>topic</mat-icon>
+                                <span i18n="Archives menu label">Archives</span>
+                            </button>
+                            <button class="top-menu-button" (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
+                            <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
+                            <span i18n="Dark mode toggle label">Dark</span>
+                            <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
+                            </button>
+                            <button class="top-menu-button" (click)="openAboutDialog()" mat-menu-item>
+                                <mat-icon>info</mat-icon>
+                                <span i18n="About menu label">About</span>
+                            </button>
+                        </mat-menu>
                     </div>
-                </div>
-                <div class="flex-column" style="text-align: right; align-items: flex-end;">
-                    <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
-                    <mat-menu #menuSettings="matMenu">
-                        <button (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
-                            <mat-icon>person</mat-icon>
-                            <span i18n="Profile menu label">Profile</span>
-                        </button>
-                        <button (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
-                          <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
-                          <span i18n="Dark mode toggle label">Dark</span>
-                          <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
-                        </button>
-                        <!-- <button *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('settings')))" (click)="openSettingsDialog()" mat-menu-item>
-                          <mat-icon>settings</mat-icon>
-                          <span i18n="Settings menu label">Settings</span>
-                        </button> -->
-                        <button (click)="openAboutDialog()" mat-menu-item>
-                            <mat-icon>info</mat-icon>
-                            <span i18n="About menu label">About</span>
-                        </button>
-                    </mat-menu>
                 </div>
             </div>
         </mat-toolbar>
@@ -44,14 +48,14 @@
                     <a *ngIf="postsService.config && postsService.config.Advanced.multi_user_mode && !postsService.isLoggedIn" mat-list-item (click)="sidenav.close()" routerLink='/login'><ng-container i18n="Navigation menu Login Page title">Login</ng-container></a>
                     <a *ngIf="postsService.config && allowSubscriptions && postsService.hasPermission('subscriptions')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a>
                     <a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a>
-                    <a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/tasks'><ng-container i18n="Navigation menu Tasks Page title">Tasks</ng-container></a>
+                    <a *ngIf="postsService.config && postsService.hasPermission('tasks_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/tasks'><ng-container i18n="Navigation menu Tasks Page title">Tasks</ng-container></a>
                     <ng-container *ngIf="postsService.config && postsService.hasPermission('settings')">
                         <mat-divider></mat-divider>
                         <a mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/settings'><ng-container i18n="Settings menu label">Settings</ng-container></a>
                     </ng-container>
                     <ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
                         <mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
-                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
+                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.display]="'inline-block'" [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
                     </ng-container>
                 </mat-nav-list>
             </mat-sidenav>

src/app/app.component.ts

@@ -20,6 +20,8 @@ import { SettingsComponent } from './settings/settings.component';
 import { AboutDialogComponent } from './dialogs/about-dialog/about-dialog.component';
 import { UserProfileDialogComponent } from './dialogs/user-profile-dialog/user-profile-dialog.component';
 import { SetDefaultAdminDialogComponent } from './dialogs/set-default-admin-dialog/set-default-admin-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 @Component({
   selector: 'app-root',
@@ -45,9 +47,12 @@ export class AppComponent implements OnInit, AfterViewInit {
   enableDownloadsManager = false;
 
   @ViewChild('sidenav') sidenav: MatSidenav;
+  @ViewChild('notifications') notifications: NotificationsComponent;
   @ViewChild('hamburgerMenu', { read: ElementRef }) hamburgerMenuButton: ElementRef;
   navigator: string = null;
 
+  notification_count = 0;
+
   constructor(public postsService: PostsService, public snackBar: MatSnackBar, private dialog: MatDialog,
     public router: Router, public overlayContainer: OverlayContainer, private elementRef: ElementRef) {
 
@@ -71,7 +76,7 @@ export class AppComponent implements OnInit, AfterViewInit {
 
   }
 
-  ngOnInit() {
+  ngOnInit(): void {
     if (localStorage.getItem('theme')) {
       this.setTheme(localStorage.getItem('theme'));
     }
@@ -90,15 +95,15 @@ export class AppComponent implements OnInit, AfterViewInit {
     });
   }
 
-  ngAfterViewInit() {
+  ngAfterViewInit(): void {
     this.postsService.sidenav = this.sidenav;
   }
 
-  toggleSidenav() {
+  toggleSidenav(): void {
     this.sidenav.toggle();
   }
 
-  loadConfig() {
+  loadConfig(): void {
     // loading config
     this.topBarTitle = this.postsService.config['Extra']['title_top'];
     const themingExists = this.postsService.config['Themes'];
@@ -164,7 +169,7 @@ export class AppComponent implements OnInit, AfterViewInit {
     this.componentCssClass = theme;
   }
 
-  flipTheme() {
+  flipTheme(): void {
     if (this.postsService.theme.key === 'default') {
       this.setTheme('dark');
     } else if (this.postsService.theme.key === 'dark') {
@@ -172,17 +177,12 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  themeMenuItemClicked(event) {
+  themeMenuItemClicked(event): void {
     this.flipTheme();
     event.stopPropagation();
   }
 
-  getSubscriptions() {
-
-  }
-
-
-  goBack() {
+  goBack(): void {
     if (!this.navigator) {
       this.router.navigate(['/home']);
     } else {
@@ -190,23 +190,41 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  openSettingsDialog() {
-    const dialogRef = this.dialog.open(SettingsComponent, {
+  openSettingsDialog(): void {
+    this.dialog.open(SettingsComponent, {
       width: '80vw'
     });
   }
 
-  openAboutDialog() {
-    const dialogRef = this.dialog.open(AboutDialogComponent, {
+  openAboutDialog(): void {
+    this.dialog.open(AboutDialogComponent, {
       width: '80vw'
     });
   }
 
-  openProfileDialog() {
-    const dialogRef = this.dialog.open(UserProfileDialogComponent, {
+  openProfileDialog(): void {
+    this.dialog.open(UserProfileDialogComponent, {
       width: '60vw'
     });
   }
 
+  openArchivesDialog(): void {
+    this.dialog.open(ArchiveViewerComponent, {
+      width: '85vw'
+    });
+  }
+
+  notificationCountUpdate(new_count: number): void {
+    this.notification_count = new_count;
+  }
+
+  notificationMenuOpened(): void {
+    this.notifications.getNotifications();
+  }
+
+  notificationMenuClosed(): void {
+    this.notifications.setNotificationsToRead();
+  }
+
 }
 

src/app/app.module.ts

@@ -29,6 +29,8 @@ import { MatPaginatorModule } from '@angular/material/paginator';
 import { MatSortModule } from '@angular/material/sort';
 import { MatTableModule } from '@angular/material/table';
 import { MatDatepickerModule } from '@angular/material/datepicker';
+import { MatChipsModule } from '@angular/material/chips';
+import { MatBadgeModule } from '@angular/material/badge';
 import { DragDropModule } from '@angular/cdk/drag-drop';
 import { ClipboardModule } from '@angular/cdk/clipboard';
 import { TextFieldModule } from '@angular/cdk/text-field';
@@ -49,10 +51,8 @@ import { CreatePlaylistComponent } from './create-playlist/create-playlist.compo
 import { SubscriptionsComponent } from './subscriptions/subscriptions.component';
 import { SubscribeDialogComponent } from './dialogs/subscribe-dialog/subscribe-dialog.component';
 import { SubscriptionComponent } from './subscription//subscription/subscription.component';
-import { SubscriptionFileCardComponent } from './subscription/subscription-file-card/subscription-file-card.component';
 import { SubscriptionInfoDialogComponent } from './dialogs/subscription-info-dialog/subscription-info-dialog.component';
 import { SettingsComponent } from './settings/settings.component';
-import { MatChipsModule } from '@angular/material/chips';
 import { NgxFileDropModule } from 'ngx-file-drop';
 import { AvatarModule } from 'ngx-avatars';
 import { ContentLoaderModule } from '@ngneat/content-loader';
@@ -74,7 +74,6 @@ import { ManageUserComponent } from './components/manage-user/manage-user.compon
 import { ManageRoleComponent } from './components/manage-role/manage-role.component';
 import { CookiesUploaderDialogComponent } from './dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component';
 import { LogsViewerComponent } from './components/logs-viewer/logs-viewer.component';
-import { ModifyPlaylistComponent } from './dialogs/modify-playlist/modify-playlist.component';
 import { ConfirmDialogComponent } from './dialogs/confirm-dialog/confirm-dialog.component';
 import { UnifiedFileCardComponent } from './components/unified-file-card/unified-file-card.component';
 import { RecentVideosComponent } from './components/recent-videos/recent-videos.component';
@@ -89,6 +88,13 @@ import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-butto
 import { TasksComponent } from './components/tasks/tasks.component';
 import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
 import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { NotificationsListComponent } from './components/notifications-list/notifications-list.component';
+import { TaskSettingsComponent } from './components/task-settings/task-settings.component';
+import { GenerateRssUrlComponent } from './dialogs/generate-rss-url/generate-rss-url.component';
+import { SortPropertyComponent } from './components/sort-property/sort-property.component';
+import { OnlyNumberDirective } from './directives/only-number.directive';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 registerLocaleData(es, 'es');
 
@@ -102,7 +108,6 @@ registerLocaleData(es, 'es');
         SubscriptionsComponent,
         SubscribeDialogComponent,
         SubscriptionComponent,
-        SubscriptionFileCardComponent,
         SubscriptionInfoDialogComponent,
         SettingsComponent,
         AboutDialogComponent,
@@ -123,7 +128,6 @@ registerLocaleData(es, 'es');
         ManageRoleComponent,
         CookiesUploaderDialogComponent,
         LogsViewerComponent,
-        ModifyPlaylistComponent,
         ConfirmDialogComponent,
         UnifiedFileCardComponent,
         RecentVideosComponent,
@@ -136,7 +140,14 @@ registerLocaleData(es, 'es');
         SkipAdButtonComponent,
         TasksComponent,
         UpdateTaskScheduleDialogComponent,
-        RestoreDbDialogComponent
+        RestoreDbDialogComponent,
+        NotificationsComponent,
+        NotificationsListComponent,
+        TaskSettingsComponent,
+        GenerateRssUrlComponent,
+        SortPropertyComponent,
+        OnlyNumberDirective,
+        ArchiveViewerComponent
     ],
     imports: [
         CommonModule,
@@ -174,6 +185,7 @@ registerLocaleData(es, 'es');
         MatTableModule,
         MatDatepickerModule,
         MatChipsModule,
+        MatBadgeModule,
         DragDropModule,
         ClipboardModule,
         TextFieldModule,

src/app/components/archive-viewer/archive-viewer.component.html

@@ -0,0 +1,143 @@
+<mat-form-field class="filter">
+  <mat-icon matPrefix>search</mat-icon>
+  <mat-label i18n="Filter">Filter</mat-label>
+  <input matInput [(ngModel)]="text_filter" (keyup)="applyFilter($event)" #input>
+</mat-form-field>
+
+<div [hidden]="!(archives && archives.length > 0)">
+    <div class="mat-elevation-z8">
+        <mat-table matSort [dataSource]="dataSource">
+
+          <!-- Select Column -->
+            <!-- Checkbox Column -->
+            <ng-container matColumnDef="select">
+              <mat-header-cell *matHeaderCellDef>
+                <mat-checkbox (change)="$event ? toggleAllRows() : null"
+                              [checked]="selection.hasValue() && isAllSelected()"
+                              [indeterminate]="selection.hasValue() && !isAllSelected()">
+                </mat-checkbox>
+              </mat-header-cell>
+              <mat-cell *matCellDef="let row">
+                <mat-checkbox (click)="$event.stopPropagation()"
+                              (change)="$event ? selection.toggle(row) : null"
+                              [checked]="selection.isSelected(row)">
+                </mat-checkbox>
+                <mat-icon class="audio-video-icon">{{(row.type === 'audio') ? 'audiotrack' : 'movie'}}</mat-icon>
+              </mat-cell>
+            </ng-container>
+      
+          <!-- Date Column -->
+          <ng-container matColumnDef="timestamp">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Date">Date</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element"> {{element.timestamp*1000 | date: 'short'}} </mat-cell>
+          </ng-container>
+      
+          <!-- Title Column -->
+          <ng-container matColumnDef="title">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="max-two-lines" [matTooltip]="element.title ? element.title : null">
+                    {{element.title}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+           <!-- ID Column -->
+           <ng-container matColumnDef="id">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="ID">ID</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.title ? element.title : null">
+                    {{element.id}}
+                </span>
+            </mat-cell>
+          </ng-container>
+
+          <!-- Extractor Column -->
+          <ng-container matColumnDef="extractor">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Extractor">Extractor</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.extractor? element.extractor : null">
+                    {{element.extractor}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+            <mat-header-row *matHeaderRowDef="displayedColumns; sticky: true"></mat-header-row>
+            <mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
+        </mat-table>
+      </div>
+</div>
+
+<div *ngIf="(!archives || archives.length === 0)">
+  <h4 style="text-align: center; margin-top: 10px;" i18n="Archives empty">Archives empty</h4>
+</div>
+
+<div style="margin: 10px 10px 10px 0px; display: flex;">
+  <span style="flex-grow: 1;" class="flex-items">
+    <button [disabled]="selection.selected.length === 0" color="warn" style="margin: 10px;" mat-stroked-button i18n="Delete selected" (click)="openDeleteSelectedArchivesDialog()">Delete selected</button>
+  </span>
+  <span class="flex-items">
+    <button [disabled]="!(archives && archives.length > 0)" (click)="downloadArchive()" mat-stroked-button i18n="Download archive">Download archive</button>
+    <mat-form-field style="width: 150px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="Subscription">Subscription</mat-label>
+      <mat-select [ngModel]="sub_id" (ngModelChange)="subFilterSelectionChanged($event)">
+        <mat-option [value]="'none'" i18n="None">None</mat-option>
+        <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+      </mat-select>
+    </mat-form-field>
+    <mat-form-field style="width: 100px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="File type">File type</mat-label>
+      <mat-select [ngModel]="type" (ngModelChange)="typeFilterSelectionChanged($event)" [disabled]="sub_id !== 'none'">
+        <mat-option [value]="'both'" i18n="Both">Both</mat-option>
+        <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+        <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+      </mat-select>
+    </mat-form-field>
+  </span>
+</div>
+
+<div class="file-drop-parent">
+  <ngx-file-drop [multiple]="false" accept=".txt" dropZoneLabel="Drop file here" (onFileDrop)="dropped($event)">
+      <ng-template class="file-drop" ngx-file-drop-content-tmp let-openFileSelector="openFileSelector">
+          <div style="text-align: center">
+              <div>
+                  <ng-container i18n="Drag and Drop">Drag and Drop</ng-container>
+              </div>
+              <div style="margin-top: 6px;">
+                  <button mat-stroked-button (click)="openFileSelector()">Browse Files</button>
+              </div>
+          </div>
+      </ng-template>
+  </ngx-file-drop>
+</div>
+
+<div style="margin-top: 10px; color: white">
+  <table class="table">
+      <tbody class="upload-name-style">
+          <tr *ngFor="let item of files; let i=index">
+              <td style="vertical-align: middle; border-top: unset">
+                  <strong>{{ item.relativePath }}</strong>
+              </td>
+              <td style="border-top: unset">
+                <div style="float: right">
+                  <mat-form-field style="width: 150px;">
+                    <mat-label i18n="Subscription">Subscription</mat-label>
+                    <mat-select [ngModel]="upload_sub_id" (ngModelChange)="subUploadFilterSelectionChanged($event)">
+                      <mat-option [value]="'none'" i18n="None">None</mat-option>
+                      <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <mat-form-field style="width: 100px; margin-left: 10px">
+                    <mat-label i18n="File type">File type</mat-label>
+                    <mat-select [(ngModel)]="upload_type" [value]="upload_type" [disabled]="upload_sub_id !== 'none'">
+                      <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+                      <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <button style="margin-left: 10px" [disabled]="uploading_archive || uploaded_archive" (click)="importArchive()" matTooltip="Upload" i18n-matTooltip="Upload" mat-mini-fab><mat-icon>publish</mat-icon><mat-spinner *ngIf="uploading_archive" class="spinner" [diameter]="38"></mat-spinner></button>
+                </div>
+              </td>
+          </tr>
+      </tbody>
+  </table>
+</div>

src/app/components/archive-viewer/archive-viewer.component.scss

@@ -0,0 +1,42 @@
+.filter {
+    width: 100%;
+}
+
+.spinner {
+    bottom: 1px;
+    left: 0.5px;
+    position: absolute;
+}
+
+.mat-mdc-table {
+    width: 100%;
+    max-height: 60vh;
+    overflow: auto;
+}
+
+.max-two-lines {
+    display: -webkit-box;
+    display: -moz-box;
+    max-height: 2.4em;
+    line-height: 1.2em;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    -webkit-box-orient: vertical;
+    -webkit-line-clamp: 2;
+}
+
+:host ::ng-deep .ngx-file-drop__content {
+    width: 100%;
+    top: -12px;
+    position: relative;
+}
+
+.file-drop-parent {
+    padding: 0px 10px 0px 10px;
+}
+
+.flex-items {
+    display: flex;
+    align-items: center;
+}
+

src/app/components/archive-viewer/archive-viewer.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ArchiveViewerComponent } from './archive-viewer.component';
+
+describe('ArchiveViewerComponent', () => {
+  let component: ArchiveViewerComponent;
+  let fixture: ComponentFixture<ArchiveViewerComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ArchiveViewerComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(ArchiveViewerComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/archive-viewer/archive-viewer.component.ts

@@ -0,0 +1,198 @@
+import { SelectionModel } from '@angular/cdk/collections';
+import { Component, ViewChild } from '@angular/core';
+import { MatDialog } from '@angular/material/dialog';
+import { MatSort } from '@angular/material/sort';
+import { MatTableDataSource } from '@angular/material/table';
+import { FileType } from 'api-types';
+import { Archive } from 'api-types/models/Archive';
+import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
+import { PostsService } from 'app/posts.services';
+import { NgxFileDropEntry } from 'ngx-file-drop';
+
+@Component({
+  selector: 'app-archive-viewer',
+  templateUrl: './archive-viewer.component.html',
+  styleUrls: ['./archive-viewer.component.scss']
+})
+export class ArchiveViewerComponent {
+  // table
+  displayedColumns: string[] = ['select', 'timestamp', 'title', 'id', 'extractor'];
+  dataSource = null;
+  selection = new SelectionModel<Archive>(true, []);
+
+  // general
+  archives = null;
+  archives_retrieved = false;
+  text_filter = '';
+  sub_id = 'none';
+  upload_sub_id = 'none';
+  type: FileType | 'both' = 'both';
+  upload_type: FileType = FileType.VIDEO;
+
+  // importing
+  uploading_archive = false;
+  uploaded_archive = false;
+  files = [];
+
+  typeSelectOptions = {
+    video: {
+      key: 'video',
+      label: $localize`Video`
+    },
+    audio: {
+      key: 'audio',
+      label: $localize`Audio`
+    }
+  };
+
+  @ViewChild(MatSort) sort: MatSort;
+  
+  constructor(public postsService: PostsService, private dialog: MatDialog) {
+
+  }
+
+  ngOnInit() {
+    this.getArchives();
+  }
+
+  applyFilter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.dataSource.filter = filterValue.trim().toLowerCase();
+  }
+
+  /** Whether the number of selected elements matches the total number of rows. */
+  isAllSelected() {
+    const numSelected = this.selection.selected.length;
+    const numRows = this.dataSource.data.length;
+    return numSelected === numRows;
+  }
+
+  /** Selects all rows if they are not all selected; otherwise clear selection. */
+  toggleAllRows() {
+    if (this.isAllSelected()) {
+      this.selection.clear();
+      return;
+    }
+
+    this.selection.select(...this.dataSource.data);
+  }
+
+  typeFilterSelectionChanged(value): void {
+    this.type = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    this.getArchives();
+  }
+
+  subFilterSelectionChanged(value): void {
+    this.sub_id = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    if (this.sub_id !== 'none') {
+      this.type = this.postsService.getSubscriptionByID(this.sub_id)['type'];
+    }
+    this.getArchives();
+  }
+
+  subUploadFilterSelectionChanged(value): void {
+    this.upload_sub_id = value;
+    if (this.upload_sub_id !== 'none') {
+      this.upload_type = this.postsService.getSubscriptionByID(this.upload_sub_id)['type'];
+    }
+  }
+
+  getArchives(): void {
+    this.postsService.getArchives(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      if (res['archives'] !== null 
+        && res['archives'] !== undefined
+        && JSON.stringify(this.archives) !== JSON.stringify(res['archives'])) {
+          this.archives = res['archives']
+          this.dataSource = new MatTableDataSource<Archive>(this.archives);
+          this.dataSource.sort = this.sort;
+      } else {
+        // failed to get downloads
+      }
+    });
+  }
+
+  importArchive(): void {
+    this.uploading_archive = true;
+        for (const droppedFile of this.files) {
+      // Is it a file?
+      if (droppedFile.fileEntry.isFile) {
+        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
+        fileEntry.file(async (file: File) => {          
+          const archive_base64 = await blobToBase64(file);
+          this.postsService.importArchive(archive_base64 as string, this.upload_type, this.upload_sub_id === 'none' ? null : this.upload_sub_id).subscribe(res => {
+            this.uploading_archive = false;
+            if (res['success']) {
+              this.uploaded_archive = true;
+              this.postsService.openSnackBar($localize`Archive successfully imported!`);
+            }
+            this.getArchives();
+          }, err => {
+            console.error(err);
+            this.uploading_archive = false;
+          });
+        });
+      }
+    }
+  }
+
+  downloadArchive(): void {
+    this.postsService.downloadArchive(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      const blob: Blob = res;
+      saveAs(blob, 'archive.txt');
+    });
+  }
+
+  openDeleteSelectedArchivesDialog(): void {
+    const dialogRef = this.dialog.open(ConfirmDialogComponent, {
+      data: {
+        dialogTitle: $localize`Delete archives`,
+        dialogText: $localize`Would you like to delete ${this.selection.selected.length}:selected archives amount: archive(s)?`,
+        submitText: $localize`Delete`,
+        warnSubmitColor: true
+      }
+    });
+    dialogRef.afterClosed().subscribe(confirmed => {
+      if (confirmed) {
+        this.deleteSelectedArchives();
+      }
+    });
+    
+  }
+
+  deleteSelectedArchives(): void {
+    for (const archive of this.selection.selected) {
+      this.archives = this.archives.filter((_archive: Archive) => !(archive['extractor'] === _archive['extractor'] && archive['id'] !== _archive['id']));
+    }
+    this.postsService.deleteArchiveItems(this.selection.selected).subscribe(res => {
+      if (res['success']) {
+        this.postsService.openSnackBar($localize`Successfully deleted archive items!`);
+      } else {
+        this.postsService.openSnackBar($localize`Failed to delete archive items!`);
+      }
+      this.getArchives();
+    });
+    this.selection.clear();
+  }
+
+  public dropped(files: NgxFileDropEntry[]) {
+    this.files = files;
+    this.uploading_archive = false;
+    this.uploaded_archive = false;
+  }
+
+  originalOrder = (): number => {
+    return 0;
+  }
+}
+
+function blobToBase64(blob: Blob) {
+  return new Promise((resolve, _) => {
+    const reader = new FileReader();
+    reader.onloadend = () => resolve(reader.result);
+    reader.readAsDataURL(blob);
+  });
+}

src/app/components/custom-playlists/custom-playlists.component.html

@@ -2,7 +2,7 @@
     <div class="container">
         <div class="row justify-content-center">
             <div *ngFor="let playlist of playlists; let i = index" class="mb-2 mt-2" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
-                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [loading]="false"></app-unified-file-card>
+                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? this.postsService.token : ''" [loading]="false"></app-unified-file-card>
             </div>
         </div>
     </div>

src/app/components/custom-playlists/custom-playlists.component.ts

@@ -3,7 +3,7 @@ import { PostsService } from 'app/posts.services';
 import { Router } from '@angular/router';
 import { MatDialog } from '@angular/material/dialog';
 import { CreatePlaylistComponent } from 'app/create-playlist/create-playlist.component';
-import { ModifyPlaylistComponent } from 'app/dialogs/modify-playlist/modify-playlist.component';
+import { Playlist } from 'api-types';
 
 @Component({
   selector: 'app-custom-playlists',
@@ -32,7 +32,7 @@ export class CustomPlaylistsComponent implements OnInit {
     });
   }
 
-  getAllPlaylists() {
+  getAllPlaylists(): void {
     this.playlists_received = false;
     // must call getAllFiles as we need to get category playlists as well
     this.postsService.getPlaylists(true).subscribe(res => {
@@ -42,22 +42,25 @@ export class CustomPlaylistsComponent implements OnInit {
   }
 
   // creating a playlist
-  openCreatePlaylistDialog() {
+  openCreatePlaylistDialog(): void {
     const dialogRef = this.dialog.open(CreatePlaylistComponent, {
       data: {
-      }
+        create_mode: true
+      },
+      minWidth: '90vw',
+      minHeight: '95vh'
     });
     dialogRef.afterClosed().subscribe(result => {
       if (result) {
         this.getAllPlaylists();
-        this.postsService.openSnackBar('Successfully created playlist!', '');
+        this.postsService.openSnackBar($localize`Successfully created playlist!`);
       } else if (result === false) {
-        this.postsService.openSnackBar('ERROR: failed to create playlist!', '');
+        this.postsService.openSnackBar($localize`ERROR: failed to create playlist!`);
       }
     });
   }
 
-  goToPlaylist(info_obj) {
+  goToPlaylist(info_obj: { file: Playlist; }): void {
     const playlist = info_obj.file;
     const playlistID = playlist.id;
 
@@ -72,11 +75,12 @@ export class CustomPlaylistsComponent implements OnInit {
       }
     } else {
       // playlist not found
+      // TODO: Make translatable
       console.error(`Playlist with ID ${playlistID} not found!`);
     }
   }
 
-  downloadPlaylist(playlist_id, playlist_name) {
+  downloadPlaylist(playlist_id: string, playlist_name: string): void {
     this.downloading_content[playlist_id] = true;
     this.postsService.downloadPlaylistFromServer(playlist_id).subscribe(res => {
       this.downloading_content[playlist_id] = false;
@@ -86,33 +90,34 @@ export class CustomPlaylistsComponent implements OnInit {
 
   }
 
-  deletePlaylist(args) {
+  deletePlaylist(args: { file: Playlist; index: number; }): void {
     const playlist = args.file;
     const index = args.index;
     const playlistID = playlist.id;
-    this.postsService.removePlaylist(playlistID, playlist.type).subscribe(res => {
+    this.postsService.removePlaylist(playlistID).subscribe(res => {
       if (res['success']) {
         this.playlists.splice(index, 1);
-        this.postsService.openSnackBar('Playlist successfully removed.', '');
+        this.postsService.openSnackBar($localize`Playlist successfully removed.`);
       }
       this.getAllPlaylists();
     });
   }
 
-  editPlaylistDialog(args) {
+  editPlaylistDialog(args: { playlist: Playlist; index: number; }): void {
     const playlist = args.playlist;
     const index = args.index;
-    const dialogRef = this.dialog.open(ModifyPlaylistComponent, {
+    const dialogRef = this.dialog.open(CreatePlaylistComponent, {
       data: {
         playlist_id: playlist.id,
-        width: '65vw'
-      }
+        create_mode: false
+      },
+      minWidth: '85vw'
     });
 
-    dialogRef.afterClosed().subscribe(res => {
+    dialogRef.afterClosed().subscribe(() => {
       // updates playlist in file manager if it changed
       if (dialogRef.componentInstance.playlist_updated) {
-        this.playlists[index] = dialogRef.componentInstance.original_playlist;
+        this.playlists[index] = dialogRef.componentInstance.playlist;
       }
     });
   }

src/app/components/downloads/downloads.component.ts

@@ -8,6 +8,7 @@ import { MatDialog } from '@angular/material/dialog';
 import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
 import { MatSort } from '@angular/material/sort';
 import { Clipboard } from '@angular/cdk/clipboard';
+import { Download } from 'api-types';
 
 @Component({
   selector: 'app-downloads',
@@ -40,7 +41,7 @@ import { Clipboard } from '@angular/cdk/clipboard';
 })
 export class DownloadsComponent implements OnInit, OnDestroy {
 
-  @Input() uids = null;
+  @Input() uids: string[] = null;
 
   downloads_check_interval = 1000;
   downloads = [];
@@ -68,7 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   @ViewChild(MatPaginator) paginator: MatPaginator;
   @ViewChild(MatSort) sort: MatSort;
 
-  sort_downloads = (a, b) => {
+  sort_downloads = (a: Download, b: Download): number => {
     const result = b.timestamp_start - a.timestamp_start;
     return result;
   }
@@ -166,7 +167,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   pauseDownload(download_uid: string): void {
     this.postsService.pauseDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause download! See server logs for more info.`);
       }
     });
   }
@@ -174,7 +175,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   pauseAllDownloads(): void {
     this.postsService.pauseAllDownloads().subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause all downloads! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause all downloads! See server logs for more info.`);
       }
     });
   }
@@ -182,7 +183,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   resumeDownload(download_uid: string): void {
     this.postsService.resumeDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to resume download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to resume download! See server logs for more info.`);
       }
     });
   }
@@ -190,7 +191,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   resumeAllDownloads(): void {
     this.postsService.resumeAllDownloads().subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to resume all downloads! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to resume all downloads! See server logs for more info.`);
       }
     });
   }
@@ -198,7 +199,11 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   restartDownload(download_uid: string): void {
     this.postsService.restartDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to restart download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to restart download! See server logs for more info.`);
+      } else {
+        if (this.uids && res['new_download_uid']) {
+          this.uids.push(res['new_download_uid']);
+        }
       }
     });
   }
@@ -206,7 +211,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   cancelDownload(download_uid: string): void {
     this.postsService.cancelDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to cancel download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to cancel download! See server logs for more info.`);
       }
     });
   }
@@ -214,12 +219,12 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   clearDownload(download_uid: string): void {
     this.postsService.clearDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause download! See server logs for more info.`);
       }
     });
   }
 
-  watchContent(download): void {
+  watchContent(download: Download): void {
     const container = download['container'];
     localStorage.setItem('player_navigator', this.router.url.split(';')[0]);
     const is_playlist = container['uids']; // hacky, TODO: fix
@@ -230,7 +235,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     }
   }
 
-  combineDownloads(downloads_old, downloads_new) {
+  combineDownloads(downloads_old: Download[], downloads_new: Download[]): Download[] {
     // only keeps downloads that exist in the new set
     downloads_old = downloads_old.filter(download_old => downloads_new.some(download_new => download_new.uid === download_old.uid));
 
@@ -251,7 +256,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     return downloads_old;
   }
 
-  showError(download) {
+  showError(download: Download): void {
     const copyToClipboardEmitter = new EventEmitter<boolean>();
     this.dialog.open(ConfirmDialogComponent, {
       data: {
@@ -272,10 +277,3 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     });
   }
 }
-
-export interface Download {
-  timestamp_start: number;
-  title: string;
-  step_index: number;
-  progress: string;
-}

src/app/components/login/login.component.html

@@ -1,31 +1,36 @@
 <mat-card class="login-card">
-    <mat-tab-group style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
+    <mat-tab-group mat-stretch-tabs style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
         <mat-tab label="Login" i18n-label="Login">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="loginUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="loginUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>
         <mat-tab *ngIf="registrationEnabled" label="Register" i18n-label="Register">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="registrationUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="registrationUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput placeholder="Confirm Password" i18n-placeholder="Confirm Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Confirm Password">Confirm Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>

src/app/components/login/login.component.scss

@@ -12,17 +12,15 @@
 }
 
 .login-button-div {
-    margin-bottom: 10px;
-    margin-top: 10px;
-    margin-left: -16px;
-    margin-right: -16px;
     bottom: 0px;
     width: 100%;
+    height: 42px;
     position: absolute;
 }
 
 .login-button-div > button {
     width: 100%;
+    height: 100%;
     border-radius: 0px 0px 4px 4px !important;
 }
 

src/app/components/logs-viewer/logs-viewer.component.ts

@@ -43,17 +43,17 @@ export class LogsViewerComponent implements OnInit {
           })
         });
       } else {
-        this.postsService.openSnackBar('Failed to retrieve logs!');
+        this.postsService.openSnackBar($localize`Failed to retrieve logs!`);
       }
     }, err => {
       this.logs_loading = false;
       console.error(err);
-      this.postsService.openSnackBar('Failed to retrieve logs!');
+      this.postsService.openSnackBar($localize`Failed to retrieve logs!`);
     });
   }
 
   copiedLogsToClipboard() {
-    this.postsService.openSnackBar('Logs copied to clipboard!');
+    this.postsService.openSnackBar($localize`Logs copied to clipboard!`);
   }
 
   clearLogs() {
@@ -72,12 +72,12 @@ export class LogsViewerComponent implements OnInit {
             this.logs = [];
             this.logs_text = '';
             this.getLogs();
-            this.postsService.openSnackBar('Logs successfully cleared!');
+            this.postsService.openSnackBar($localize`Logs successfully cleared!`);
           } else {
-            this.postsService.openSnackBar('Failed to clear logs!');
+            this.postsService.openSnackBar($localize`Failed to clear logs!`);
           }
         }, err => {
-          this.postsService.openSnackBar('Failed to clear logs!');
+          this.postsService.openSnackBar($localize`Failed to clear logs!`);
         });
       }
     });

src/app/components/manage-role/manage-role.component.html

@@ -1,17 +1,15 @@
-<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.name}}</h4>
+<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.key}}</h4>
 
 <mat-dialog-content *ngIf="role">
-    <mat-list>
-        <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
-            <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
-            <span matLine>
-                <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
-                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
-                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
-                </mat-radio-group>
-            </span>
-        </mat-list-item>
-    </mat-list>
+    <div *ngFor="let permission of available_permissions">
+        <div matListItemTitle>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</div>
+        <div matListItemLine>
+            <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
+                <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
+                <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
+            </mat-radio-group>
+        </div>
+    </div>
 </mat-dialog-content>
 
 <mat-dialog-actions>

src/app/components/manage-role/manage-role.component.scss

@@ -1,4 +0,0 @@
-.mat-radio-button {
-    margin-right: 10px;
-    margin-top: 5px;
-}

src/app/components/manage-role/manage-role.component.ts

@@ -14,16 +14,17 @@ export class ManageRoleComponent implements OnInit {
   permissions = null;
 
   permissionToLabel = {
-    'filemanager': 'File manager',
-    'settings': 'Settings access',
-    'subscriptions': 'Subscriptions',
-    'sharing': 'Share files',
-    'advanced_download': 'Use advanced download mode',
-    'downloads_manager': 'Use downloads manager'
+    'filemanager': $localize`File manager`,
+    'settings': $localize`Settings access`,
+    'subscriptions': $localize`Subscriptions`,
+    'sharing': $localize`Share files`,
+    'advanced_download': $localize`Use advanced download mode`,
+    'downloads_manager': $localize`Use downloads manager`,
+    'tasks_manager': $localize`Use tasks manager`,
   }
 
   constructor(public postsService: PostsService, private dialogRef: MatDialogRef<ManageRoleComponent>,
-              @Inject(MAT_DIALOG_DATA) public data: any) {
+              @Inject(MAT_DIALOG_DATA) public data: {role: string}) {
     if (this.data) {
       this.role = this.data.role;
       this.available_permissions = this.postsService.available_permissions;

src/app/components/manage-user/manage-user.component.html

@@ -5,24 +5,23 @@
 
     <div>
         <mat-form-field style="margin-right: 15px;">
-            <input matInput [(ngModel)]="newPasswordInput" type="password" placeholder="New password" i18n-placeholder="New password placeholder">
+            <mat-label i18n="New password">New password</mat-label>
+            <input matInput [(ngModel)]="newPasswordInput" type="password">
         </mat-form-field>
         <button mat-raised-button color="accent" (click)="setNewPassword()" [disabled]="newPasswordInput.length === 0"><ng-container i18n="Set new password">Set new password</ng-container></button>
     </div>
 
     <div>
-        <mat-list>
-            <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
-                <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
-                <span matLine>
-                    <mat-radio-group [disabled]="permission === 'settings' && postsService.user.uid === user.uid" (change)="changeUserPermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give user permission for ' + permission">
-                        <mat-radio-button value="default"><ng-container i18n="Use role default">Use role default</ng-container></mat-radio-button>
-                        <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
-                        <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
-                    </mat-radio-group>
-                </span>
-            </mat-list-item>
-        </mat-list>
+        <div *ngFor="let permission of available_permissions">
+            <div matListItemTitle>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</div>
+            <div matListItemLine>
+                <mat-radio-group [disabled]="permission === 'settings' && postsService.user.uid === user.uid" (change)="changeUserPermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give user permission for ' + permission">
+                    <mat-radio-button value="default"><ng-container i18n="Use role default">Use role default</ng-container></mat-radio-button>
+                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
+                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
+                </mat-radio-group>
+            </div>
+        </div>
     </div>
 </mat-dialog-content>
 

src/app/components/manage-user/manage-user.component.scss

@@ -1,4 +1,4 @@
-.mat-radio-button {
+.mat-mdc-radio-button {
     margin-right: 10px;
     margin-top: 5px;
 }

src/app/components/manage-user/manage-user.component.ts

@@ -1,6 +1,7 @@
 import { Component, OnInit, Inject } from '@angular/core';
 import { PostsService } from 'app/posts.services';
 import { MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { User } from 'api-types';
 
 @Component({
   selector: 'app-manage-user',
@@ -15,17 +16,18 @@ export class ManageUserComponent implements OnInit {
   permissions = null;
 
   permissionToLabel = {
-    'filemanager': 'File manager',
-    'settings': 'Settings access',
-    'subscriptions': 'Subscriptions',
-    'sharing': 'Share files',
-    'advanced_download': 'Use advanced download mode',
-    'downloads_manager': 'Use downloads manager'
+    'filemanager': $localize`File manager`,
+    'settings': $localize`Settings access`,
+    'subscriptions': $localize`Subscriptions`,
+    'sharing': $localize`Share files`,
+    'advanced_download': $localize`Use advanced download mode`,
+    'downloads_manager': $localize`Use downloads manager`,
+    'tasks_manager': $localize`Use tasks manager`,
   }
 
   settingNewPassword = false;
 
-  constructor(public postsService: PostsService, @Inject(MAT_DIALOG_DATA) public data: any) {
+  constructor(public postsService: PostsService, @Inject(MAT_DIALOG_DATA) public data: {user: User}) {
     if (this.data) {
       this.user = this.data.user;
       this.available_permissions = this.postsService.available_permissions;
@@ -53,14 +55,14 @@ export class ManageUserComponent implements OnInit {
   }
 
   changeUserPermissions(change, permission) {
-    this.postsService.setUserPermission(this.user.uid, permission, change.value).subscribe(res => {
+    this.postsService.setUserPermission(this.user.uid, permission, change.value).subscribe(() => {
       // console.log(res);
     });
   }
 
   setNewPassword() {
     this.settingNewPassword = true;
-    this.postsService.changeUserPassword(this.user.uid, this.newPasswordInput).subscribe(res => {
+    this.postsService.changeUserPassword(this.user.uid, this.newPasswordInput).subscribe(() => {
       this.newPasswordInput = '';
       this.settingNewPassword = false;
     });

src/app/components/modify-users/modify-users.component.html

@@ -1,14 +1,15 @@
 <div *ngIf="dataSource; else loading">
     <div style="padding: 15px">
     <div class="row">
-        <div class="table table-responsive pb-4 pt-2">
+        <div class="table table-responsive pb-4 pt-4">
         <div class="example-header">
-            <mat-form-field>
-            <input matInput (keyup)="applyFilter($event.target.value)" placeholder="Search" i18n-placeholder="search field description">
+            <mat-form-field appearance="outline">
+                <mat-label i18n="Search">Search</mat-label>
+                <input matInput (keyup)="applyFilter($event)">
             </mat-form-field>
         </div>
         
-        <div class="example-container mat-elevation-z8">
+        <div class="mat-elevation-z8" style="margin-right: 15px;">
         
             <mat-table #table [dataSource]="dataSource" matSort>
         

src/app/components/modify-users/modify-users.component.ts

@@ -8,6 +8,7 @@ import { MatDialog, MatDialogRef } from '@angular/material/dialog';
 import { AddUserDialogComponent } from 'app/dialogs/add-user-dialog/add-user-dialog.component';
 import { ManageUserComponent } from '../manage-user/manage-user.component';
 import { ManageRoleComponent } from '../manage-role/manage-role.component';
+import { User } from 'api-types';
 
 @Component({
   selector: 'app-modify-users',
@@ -31,7 +32,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
 
   // MatPaginator Output
   pageEvent: PageEvent;
-  users: any;
+  users: User[];
   editObject = null;
   constructedObject = {};
   roles = null;
@@ -62,7 +63,8 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     this.pageSizeOptions = setPageSizeOptionsInput.split(',').map(str => +str);
   }
 
-  applyFilter(filterValue: string) {
+  applyFilter(event: KeyboardEvent) {
+    let filterValue = (event.target as HTMLInputElement).value; // "as HTMLInputElement" is required: https://angular.io/guide/user-input#type-the-event
     filterValue = filterValue.trim(); // Remove whitespace
     filterValue = filterValue.toLowerCase(); // Datasource defaults to lowercase matches
     this.dataSource.filter = filterValue;
@@ -94,11 +96,9 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     });
   }
 
-  finishEditing(user_uid) {
-    let has_finished = false;
+  finishEditing(user_uid: string) {
     if (this.constructedObject && this.constructedObject['name'] && this.constructedObject['role']) {
       if (!isEmptyOrSpaces(this.constructedObject['name']) && !isEmptyOrSpaces(this.constructedObject['role'])) {
-        has_finished = true;
         const index_of_object = this.indexOfUser(user_uid);
         this.users[index_of_object] = this.constructedObject;
         this.constructedObject = {};
@@ -109,7 +109,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     }
   }
 
-  enableEditMode(user_uid) {
+  enableEditMode(user_uid: string) {
     if (this.uidInUserList(user_uid) && this.indexOfUser(user_uid) > -1) {
       const users_index = this.indexOfUser(user_uid);
       this.editObject = this.users[users_index];
@@ -124,7 +124,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   // checks if user is in users array by name
-  uidInUserList(user_uid) {
+  uidInUserList(user_uid: string) {
     for (let i = 0; i < this.users.length; i++) {
       if (this.users[i].uid === user_uid) {
         return true;
@@ -134,7 +134,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   // gets index of user in users array by name
-  indexOfUser(user_uid) {
+  indexOfUser(user_uid: string) {
     for (let i = 0; i < this.users.length; i++) {
       if (this.users[i].uid === user_uid) {
         return i;
@@ -144,12 +144,12 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
   }
 
   setUser(change_obj) {
-    this.postsService.changeUser(change_obj).subscribe(res => {
+    this.postsService.changeUser(change_obj).subscribe(() => {
       this.getArray();
     });
   }
 
-  manageUser(user_uid) {
+  manageUser(user_uid: string) {
     const index_of_object = this.indexOfUser(user_uid);
     const user_obj = this.users[index_of_object];
     this.dialog.open(ManageUserComponent, {
@@ -160,17 +160,17 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     });
   }
 
-  removeUser(user_uid) {
-    this.postsService.deleteUser(user_uid).subscribe(res => {
+  removeUser(user_uid: string) {
+    this.postsService.deleteUser(user_uid).subscribe(() => {
       this.getArray();
-    }, err => {
+    }, () => {
       this.getArray();
     });
   }
 
   createAndSortData() {
     // Sorts the data by last finished
-    this.users.sort((a, b) => b.name > a.name);
+    this.users.sort((a, b) => a.name.localeCompare(b.name));
 
     const filteredData = [];
     for (let i = 0; i < this.users.length; i++) {
@@ -188,7 +188,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
       }
     });
 
-    dialogRef.afterClosed().subscribe(success => {
+    dialogRef.afterClosed().subscribe(() => {
       this.getRoles();
     });
   }
@@ -197,7 +197,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
     this.dialogRef.close();
   }
 
-  public openSnackBar(message: string, action: string = '') {
+  public openSnackBar(message: string, action = '') {
     this.snackBar.open(message, action, {
       duration: 2000,
     });

src/app/components/notifications-list/notifications-list.component.html

@@ -0,0 +1,30 @@
+<div class="card-radius mat-elevation-z2" *ngFor="let notification of notifications; let i = index;">
+    <mat-card class="notification-card card-radius">
+        <mat-card-header>
+            <mat-card-subtitle>
+                <div>
+                    <span class="notification-timestamp">{{notification.timestamp * 1000 | date:'short'}}</span>
+                </div>
+            </mat-card-subtitle>
+            <mat-card-title>
+                <ng-container *ngIf="NOTIFICATION_PREFIX[notification.type]">
+                    {{NOTIFICATION_PREFIX[notification.type]}}
+                </ng-container>
+            </mat-card-title>
+        </mat-card-header>
+        <mat-card-content>
+            <ng-container *ngIf="NOTIFICATION_SUFFIX_KEY[notification.type]">
+                <div style="word-break: break-word">
+                    {{notification['data'][NOTIFICATION_SUFFIX_KEY[notification.type]]}}
+                </div>
+            </ng-container>
+        </mat-card-content>
+        <mat-card-actions *ngIf="notification.actions?.length > 0">
+            <button matTooltip="Remove" i18n-matTooltip="Remove" (click)="emitDeleteNotification(notification.uid)" mat-icon-button><mat-icon>close</mat-icon></button>
+            <span *ngFor="let action of notification.actions">
+                <button [matTooltip]="NOTIFICATION_ACTION_TO_STRING[action]" (click)="emitNotificationAction(notification, action)" mat-icon-button><mat-icon>{{NOTIFICATION_ICON[action]}}</mat-icon></button>
+            </span>
+        </mat-card-actions>
+        <span *ngIf="!notification.read" class="dot"></span>
+    </mat-card>
+</div>

src/app/components/notifications-list/notifications-list.component.scss

@@ -0,0 +1,33 @@
+.notification-divider {
+    margin-bottom: 10px;
+    margin-top: 10px;
+}
+
+.notification-text {
+    margin-left: 10px;
+    margin-right: 10px;
+    display: inline-block;
+}
+
+.notification-timestamp {
+    font-size: 14px;
+}
+
+.notification-card {
+    margin-top: 5px;
+}
+
+.card-radius {
+    border-radius: 12px;
+}
+
+.dot {
+    height: 8px;
+    width: 8px;
+    background-color: red;
+    border-radius: 50%;
+    display: inline-block;
+    position: absolute;
+    right: 8px;
+    top: 8px;
+  }

src/app/components/notifications-list/notifications-list.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { NotificationsListComponent } from './notifications-list.component';
+
+describe('NotificationsListComponent', () => {
+  let component: NotificationsListComponent;
+  let fixture: ComponentFixture<NotificationsListComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ NotificationsListComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(NotificationsListComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/notifications-list/notifications-list.component.ts

@@ -0,0 +1,57 @@
+import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { Notification } from 'api-types';
+import { NotificationAction } from 'api-types/models/NotificationAction';
+import { NotificationType } from 'api-types/models/NotificationType';
+
+@Component({
+  selector: 'app-notifications-list',
+  templateUrl: './notifications-list.component.html',
+  styleUrls: ['./notifications-list.component.scss']
+})
+export class NotificationsListComponent {
+  @Input() notifications = null;
+  @Output() deleteNotification = new EventEmitter<string>();
+  @Output() notificationAction = new EventEmitter<{notification: Notification, action: NotificationAction}>();
+
+  NOTIFICATION_PREFIX: { [key in NotificationType]: string } = {
+    download_complete: $localize`Finished downloading`,
+    download_error: $localize`Download failed`,
+    task_finished: $localize`Task finished`
+  }
+
+  // Attaches string to the end of the notification text
+  NOTIFICATION_SUFFIX_KEY: { [key in NotificationType]: string } = {
+    download_complete: 'file_title',
+    download_error: 'download_url',
+    task_finished: 'task_title'
+  }
+
+  NOTIFICATION_ACTION_TO_STRING: { [key in NotificationAction]: string } = {
+    play: $localize`Play`,
+    retry_download: $localize`Retry download`,
+    view_download_error: $localize`View error`,
+    view_tasks: $localize`View task`
+  }
+
+  NOTIFICATION_COLOR: { [key in NotificationAction]: string } = {
+    play: 'primary',
+    retry_download: 'primary',
+    view_download_error: 'warn',
+    view_tasks: 'primary'
+  }
+
+  NOTIFICATION_ICON: { [key in NotificationAction]: string } = {
+    play: 'smart_display',
+    retry_download: 'restart_alt',
+    view_download_error: 'warning',
+    view_tasks: 'task'
+  }
+
+  emitNotificationAction(notification: Notification, action: NotificationAction): void {
+    this.notificationAction.emit({notification: notification, action: action});
+  }
+
+  emitDeleteNotification(uid: string): void {
+    this.deleteNotification.emit(uid);
+  }
+}

src/app/components/notifications/notifications.component.css

@@ -0,0 +1,10 @@
+.notification-title {
+    margin-bottom: 6px;
+    text-align: center
+}
+
+.notifications-list-parent {
+    max-height: 70vh;
+    overflow-y: auto;
+    padding: 0px 10px 10px 10px;
+}