Merge branch 'gh-pages' of https://github.com/Tzahi12345/YoutubeDL-Material into gh-pages

Pre 4.3.1 bug fixes

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
       - name: setup node
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
           cache: 'npm'
       - name: install dependencies
         run: |

.github/workflows/docker-release.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Set image tag
         id: tags
         run: |
-          if [ ${{ github.event.action }} == "workflow_dispatch" ]; then
+          if [ "${{ github.event.inputs.tags }}" != "" ]; then
             echo "::set-output name=tags::${{ github.event.inputs.tags }}"
           elif [ ${{ github.event.action }} == "release" ]; then
             echo "::set-output name=tags::${{ github.event.release.tag_name }}"
@@ -53,8 +53,8 @@ jobs:
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
             ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
           tags: |
-            raw=${{ steps.tags.outputs.tags }}
-            raw=latest
+            type=raw,value=${{ steps.tags.outputs.tags }}
+            type=raw,value=latest
 
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1

.github/workflows/docker.yml

@@ -13,6 +13,9 @@ on:
       - '**.pem'
       - '.dockerignore'
       - '.gitignore'
+  schedule:
+    - cron:  '34 4 * * 2'
+  workflow_dispatch:
 
 jobs:
   build-and-push:

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+    "recommendations": [
+        "angular.ng-template",
+        "dbaeumer.vscode-eslint",
+        "waderyan.gitblame",
+        "42crunch.vscode-openapi",
+        "redhat.vscode-yaml",
+        "christian-kohler.npm-intellisense",
+        "hbenl.vscode-mocha-test-adapter"
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "mochaExplorer.files": "backend/test/**/*.js",
+    "mochaExplorer.cwd": "backend",
+    "mochaExplorer.globImplementation": "vscode",
+    "mochaExplorer.env": {
+        "YTDL_MODE": "debug"
+    }
+}

.vscode/tasks.json

@@ -1,25 +1,60 @@
 {
 	"version": "2.0.0",
+	"windows": {
+        "options": {
+            "shell": {
+                "executable": "cmd.exe",
+                "args": [
+                    "/d", "/c"
+                ]
+            }
+        }
+    },
 	"tasks": [
 		{
 			"type": "npm",
 			"script": "start",
 			"problemMatcher": [],
 			"label": "Dev: start frontend",
-			"detail": "ng serve"
+			"detail": "ng serve",
+			"presentation": {
+				"echo": true,
+				"reveal": "always",
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
+			}
 		},
 		{
 			"label": "Dev: start backend",
 			"type": "shell",
-			"command": "set YTDL_MODE=debug && node app.js",
+			"command": "node app.js",
 			"options": {
-				"cwd": "./backend"
+				"cwd": "./backend",
+				"env": {
+					"YTDL_MODE": "debug"
+				}
 			},
 			"presentation": {
+				"echo": true,
 				"reveal": "always",
-				"panel": "new"
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
 			},
-			"problemMatcher": []
+			"problemMatcher": [],
+			"dependsOn": ["Dev: post-build"]
+		},
+		{
+			"label": "Dev: post-build",
+			"type": "shell",
+			"command": "node src/postbuild.mjs"
+		},
+		{
+			"label": "Dev: run all",
+			"dependsOn": ["Dev: start backend", "Dev: start frontend"]
 		}
 	]
 }

Dockerfile

@@ -2,25 +2,28 @@
 FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY ffmpeg-fetch.sh .
+COPY docker-utils/ffmpeg-fetch.sh .
+RUN chmod +x ffmpeg-fetch.sh
 RUN sh ./ffmpeg-fetch.sh
 
 
-# Create our Ubuntu 22.04 with node 16
+# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
 # Go to 20.04
 FROM ubuntu:20.04 AS base
-ENV DEBIAN_FRONTEND=noninteractive
+ARG DEBIAN_FRONTEND=noninteractive
 ENV UID=1000
 ENV GID=1000
 ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
 ENV PM2_HOME=/app/pm2
+ENV ALLOW_CONFIG_MUTATIONS=true
 RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
     apt update && \
-    apt install -y --no-install-recommends curl ca-certificates && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
     curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
     apt install -y --no-install-recommends nodejs && \
-    npm -g install npm && \
+    npm -g install npm n && \
+    n 16.14.2 && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -44,23 +47,35 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
 RUN npm install -g pm2 && \
     apt update && \
-    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 atomicparsley && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
+RUN pip install pycryptodomex
 WORKDIR /app
 # User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
 COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
+RUN chown $UID:$GID .
+RUN chmod +x /app/fix-scripts/*.sh
 # Add some persistence data
 #VOLUME ["/app/appdata"]
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime","--raw","pm2.config.js" ]
+CMD [ "npm","start" ]

Dockerfile.heroku

@@ -1,2 +1,2 @@
-FROM tzahi12345/youtubedl-material:nightly
-CMD [ "pm2-runtime", "pm2.config.js" ]
+FROM tzahi12345/youtubedl-material:latest
+CMD [ "npm", "start" ]

Public API v1.yaml

@@ -97,6 +97,11 @@ paths:
       summary: Get all files
       description: Gets all files and playlists stored in the db
       operationId: get-getAllFiles
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GetAllFilesRequest'
       responses:
         '200':
           description: OK
@@ -106,6 +111,37 @@ paths:
                 $ref: '#/components/schemas/GetAllFilesResponse'
       security:
         - Auth query parameter: []
+  /api/rss:
+    get:
+      tags:
+        - files
+      summary: Generates an RSS feed
+      description: Generates an RSS feed for downloaded files
+      operationId: get-rss
+      parameters:
+        - in: query
+          name: params
+          schema:
+            allOf:
+            - $ref: '#/components/schemas/GetAllFilesRequest'
+            - type: object
+              properties:
+                uuid:
+                  type: string
+                  description: user uid
+                  default: null
+          style: form
+          explode: true
+      responses:
+        '200':
+          description: OK
+          content:
+            text/plain:
+              schema:
+                type: string
+                description: RSS feed
+      security:
+        - Auth query parameter: []
   /api/getFile:
     post:
       tags:
@@ -542,6 +578,69 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
+  /api/deleteArchiveItems:
+    post:
+      tags:
+        - archive
+      summary: Delete item from archive
+      description: 'Deletes an item from the archive'
+      operationId: post-api-deleteArchiveItems
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DeleteArchiveItemsRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/importArchive:
+    post:
+      tags:
+        - archive
+      summary: Imports archive
+      description: 'Imports an existing archive.txt file'
+      operationId: post-api-importArchive
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ImportArchiveRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/uploadCookies:
+    post:
+      tags:
+        - downloader
+      summary: Upload cookies
+      description: 'Uploads cookies file to be used during downloading'
+      operationId: post-api-uploadCookies
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadCookiesRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -806,7 +905,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/SuccessObject'
+                $ref: '#/components/schemas/RestartDownloadResponse'
       requestBody:
         content:
           application/json:
@@ -1584,6 +1683,10 @@ components:
           type: string
           description: Height of the video, if known
           example: '1080'
+        maxHeight:
+          type: string
+          description: Max height that should be used, useful for playlists. selectedHeight will override this.
+          example: '1080'
         maxBitrate:
           type: string
           description: Specify ffmpeg/avconv audio quality
@@ -1592,6 +1695,9 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
+        ignoreArchive:
+          type: boolean
+          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -1616,6 +1722,13 @@ components:
       properties:
         download:
           $ref: '#/components/schemas/Download'
+    RestartDownloadResponse:
+      allOf:
+        - $ref: '#/components/schemas/SuccessObject'
+        - type: object
+          properties:
+            new_download_uid:
+              type: string
     GetAllDownloadsRequest:
       type: object
       properties:
@@ -1668,6 +1781,16 @@ components:
       required:
         - task_key
         - new_data
+    UpdateTaskOptionsRequest:
+      type: object
+      properties:
+        task_key:
+          type: string
+        new_options:
+          type: object
+      required:
+        - task_key
+        - new_options
     GetTaskResponse:
       type: object
       properties:
@@ -1724,6 +1847,51 @@ components:
           description: All video playlists
           items:
             $ref: '#/components/schemas/Playlist'
+    GetAllFilesRequest:
+      type: object
+      properties:
+        sort:
+          $ref: '#/components/schemas/Sort'
+        range:
+          type: array
+          items:
+            type: number
+            description: Two elements allowed, start index and end index
+          minItems: 2
+          maxItems: 2
+          default: null
+        text_search:
+          type: string
+          description: Filter files by title
+          default: null
+        file_type_filter:
+          $ref: '#/components/schemas/FileTypeFilter'
+        favorite_filter:
+          type: boolean
+          description: If set to true, only gets favorites
+          default: false
+        sub_id:
+          type: string
+          description: Include if you want to filter by subscription
+          default: null
+    Sort:
+      type: object
+      properties:
+        by:
+          type: string
+          description: Property to sort by
+          default: registered
+        order:
+          type: number
+          description: 1 for ascending, -1 for descending
+          default: -1
+    FileTypeFilter:
+      type: string
+      enum:
+        - audio_only
+        - video_only
+        - both
+      default: both
     GetAllFilesResponse:
       required:
         - files
@@ -1786,7 +1954,6 @@ components:
       required:
         - name
         - url
-        - streamingOnly
       type: object
       properties:
         name:
@@ -1842,16 +2009,11 @@ components:
           description: Number of files removed
     DeleteSubscriptionFileRequest:
       required:
-        - file
-        - sub
+        - file_uid
       type: object
       properties:
-        file:
-          type: string
         file_uid:
           type: string
-        sub:
-          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteForever:
           type: boolean
           description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -1899,7 +2061,6 @@ components:
         - uids
         - playlistName
         - thumbnailURL
-        - type
       type: object
       properties:
         playlistName:
@@ -1908,8 +2069,6 @@ components:
           type: array
           items:
             type: string
-        type:
-          $ref: '#/components/schemas/FileType'
         thumbnailURL:
           type: string
     CreatePlaylistResponse:
@@ -1939,15 +2098,17 @@ components:
       required:
         - playlist
         - success
-        - type
       type: object
       properties:
         playlist:
           $ref: '#/components/schemas/Playlist'
-        type:
-          $ref: '#/components/schemas/FileType'
         success:
           type: boolean
+        file_objs:
+          type: array
+          description: File objects for every uid in the playlist's uids property, in the same order
+          items:
+            $ref: '#/components/schemas/DatabaseFile'
     GetPlaylistsRequest:
       type: object
       properties:
@@ -1972,13 +2133,10 @@ components:
     DeletePlaylistRequest:
       required:
         - playlist_id
-        - type
       type: object
       properties:
         playlist_id:
           type: string
-        type:
-          $ref: '#/components/schemas/FileType'
     DownloadFileRequest:
       type: object
       properties:
@@ -1995,17 +2153,83 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      required:
-        - sub
       type: object
       properties:
-        sub:
-          required:
-            - archive_dir
-          type: object
-          properties:
-            archive_dir:
-              type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    Archive:
+      required:
+        - extractor
+        - id
+        - type
+        - title
+        - timestamp
+        - uid
+      type: object
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        title:
+          type: string
+        user_uid:
+          type: string
+        sub_id:
+          type: string
+        timestamp:
+          type: number
+        uid:
+          type: string
+    DeleteArchiveItemsRequest:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    ImportArchiveRequest:
+      type: object
+      required:
+        - archive
+        - type
+      properties:
+        archive:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesRequest:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesResponse:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    UploadCookiesRequest:
+      type: object
+      required:
+        - cookies
+      properties:
+        cookies:
+          type: string
+          format: binary
     UpdaterStatus:
       required:
         - details
@@ -2026,8 +2250,6 @@ components:
         tag:
           type: string
     DBInfoResponse:
-      required:
-        - db_info
       type: object
       properties:
         using_local_db:
@@ -2049,6 +2271,8 @@ components:
               $ref: '#/components/schemas/TableInfo'
             download_queue:
               $ref: '#/components/schemas/TableInfo'
+            archives:
+              $ref: '#/components/schemas/TableInfo'
     TransferDBResponse:
       required:
         - success
@@ -2348,6 +2572,7 @@ components:
         - upload_date
         - uploader
         - url
+        - favorite
       type: object
       properties:
         id:
@@ -2377,6 +2602,8 @@ components:
           type: string
         uid:
           type: string
+        user_uid:
+          type: string
         sharingEnabled:
           type: boolean
         category:
@@ -2385,6 +2612,18 @@ components:
           type: number
         local_view_count:
           type: number
+        sub_id:
+          type: string
+        registered:
+          type: number
+        height:
+          type: number
+          description: In pixels, only for videos
+        abr:
+          type: number
+          description: In Kbps
+        favorite:
+          type: boolean
     Playlist:
       required:
         - uids
@@ -2416,6 +2655,8 @@ components:
           type: string
         auto:
           type: boolean
+        sharingEnabled:
+          type: boolean
     Download:
       required:
         - url
@@ -2460,12 +2701,18 @@ components:
           type: string
           description: Error text, set if download fails.
           nullable: true
+        error_type:
+          type: string
+          description: Error type, may or may not be set in case of an error
+          nullable: true
         user_uid:
           type: string
         sub_id:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key
@@ -2480,6 +2727,8 @@ components:
       properties:
         key:
           type: string
+        title:
+          type: string
         last_ran:
           type: number
         last_confirmed:
@@ -2494,6 +2743,8 @@ components:
           type: string
         schedule:
           type: object
+        options:
+          type: object
     Schedule:
       required:
         - type
@@ -2518,6 +2769,8 @@ components:
               type: number
             timestamp:
               type: number
+            tz:
+              type: string
     DBBackup:
       required:
         - name
@@ -2560,7 +2813,6 @@ components:
         - url
         - type
         - user_uid
-        - streamingOnly
         - isPlaylist
         - videos
       type: object
@@ -2576,8 +2828,6 @@ components:
         user_uid:
           type: string
           nullable: true
-        streamingOnly:
-          type: boolean
         isPlaylist:
           type: boolean
         archive:
@@ -2709,6 +2959,44 @@ components:
           type: string
         date:
           type: string
+    Notification:
+      required:
+        - uid
+        - type
+        - text
+        - read
+        - timestamp
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/NotificationType'
+        uid:
+          type: string
+        user_uid:
+          type: string
+        action:
+          type: array
+          items:
+            $ref: '#/components/schemas/NotificationAction'
+        read:
+          type: boolean
+        data:
+          type: object
+        timestamp:
+          type: number
+    NotificationAction:
+      type: string
+      enum:
+      - play
+      - retry_download
+      - view_download_error
+      - view_tasks
+    NotificationType:
+      type: string
+      enum:
+      - download_complete
+      - download_error
+      - task_finished
     BaseChangePermissionsRequest:
       required:
         - permission
@@ -2840,6 +3128,29 @@ components:
                   type: array
                   items:
                     $ref: '#/components/schemas/UserPermission'
+    DeleteNotificationRequest:
+      required:
+        - uid
+      type: object
+      properties:
+        uid:
+          type: string
+    SetNotificationsToReadRequest:
+      required:
+        - uids
+      type: object
+      properties:
+        uids:
+          type: array
+          items:
+            type: string
+    GetNotificationsResponse:
+      type: object
+      properties:
+        notifications:
+          type: array
+          items:
+            $ref: '#/components/schemas/Notification'
   securitySchemes:
     Auth query parameter:
       name: apiKey

README.md

@@ -6,25 +6,15 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
 <hr>
 
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
-Check out the prerequisites, and go to the installation section. Easy as pie!
+Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
@@ -58,9 +48,12 @@ sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 Optional dependencies:
 
 * AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
+If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
+
 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.
@@ -91,7 +84,7 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ### Host-specific instructions
 
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
 
 ### Setup
 
@@ -102,8 +95,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for
 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
 
-NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
-
 ### Custom UID/GID
 
 By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
@@ -114,6 +105,12 @@ environment:
     GID: YOUR_GID
 ```
 
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
 ## API
 
 [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)

SECURITY.md

@@ -2,16 +2,16 @@
 
 ## Supported Versions
 
-Currently all work on this project goes into the nightly builds.
-4.2's RELEASE build is now quite old and should be considered legacy.
-We urge users to use the nightly releases, because the project
-constantly sees fixes.
+If you would like to see the latest updates, use the `nightly` tag on Docker.
 
-| Version       | Supported          |
-| ------------- | ------------------ |
-| 4.2 Nightlies | :white_check_mark: |
-| 4.2 Release   | :x:                |
-| < 4.2         | :x:                |
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
+
+| Version              | Supported          |
+| -------------------- | ------------------ |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
 
 ## Reporting a Vulnerability
 

angular.json

@@ -182,7 +182,6 @@
       }
     }
   },
-  "defaultProject": "youtube-dl-material",
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",
@@ -191,5 +190,8 @@
     "@schematics/angular:directive": {
       "prefix": "app"
     }
+  },
+  "cli": {
+    "analytics": false
   }
 }

backend/app.js

@@ -18,6 +18,7 @@ const URL = require('url').URL;
 const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
+const Feed = require('feed').Feed;
 
 // needed if bin/details somehow gets deleted
 if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -32,6 +33,7 @@ const subscriptions_api = require('./subscriptions');
 const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 var app = express();
 
@@ -68,7 +70,9 @@ db.defaults(
         configWriteFlag: false,
         downloads: {},
         subscriptions: [],
-        files_to_db_migration_complete: false
+        files_to_db_migration_complete: false,
+        tasks_manager_role_migration_complete: false,
+        archives_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -101,7 +105,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -148,22 +151,18 @@ if (fs.existsSync('version.json')) {
 
 // don't overwrite config if it already happened.. NOT
 // let alreadyWritten = db.get('configWriteFlag').value();
-let writeConfigMode = process.env.write_ytdl_config;
 
 // checks if config exists, if not, a config is auto generated
 config_api.configExistsCheck();
 
-if (writeConfigMode) {
-    setAndLoadConfig();
-} else {
-    loadConfig();
-}
+setAndLoadConfig();
 
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
 // use passport
 app.use(auth_api.passport.initialize());
+app.use(auth_api.passport.session());
 
 // actual functions
 
@@ -188,13 +187,31 @@ async function checkMigrations() {
     if (!new_db_system_migration_complete) {
         logger.info('Beginning migration: 4.2->4.3+')
         let success = await db_api.importJSONToDB(db.value(), users_db.value());
-
+        await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first
         // sets migration to complete
         db.set('new_db_system_migration_complete', true).write();
         if (success) { logger.info('4.2->4.3+ migration complete!'); }
         else { logger.error('Migration failed: 4.2->4.3+'); }
     }
 
+    const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value();
+    if (!tasks_manager_role_migration_complete) {
+        logger.info('Checking if tasks manager role permissions exist for admin user...');
+        const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes');
+        if (success) logger.info('Task manager permissions check complete!');
+        else logger.error('Failed to auto add tasks manager permissions to admin role!');
+        db.set('tasks_manager_role_migration_complete', true).write();
+    }
+
+    const archives_migration_complete = db.get('archives_migration_complete').value();
+    if (!archives_migration_complete) {
+        logger.info('Checking if archives have been migrated...');
+        const imported_archives = await archive_api.importArchives();
+        if (imported_archives) logger.info('Archives migration complete!');
+        else logger.error('Failed to migrate archives!');
+        db.set('archives_migration_complete', true).write();
+    }
+
     return true;
 }
 
@@ -484,8 +501,9 @@ async function setAndLoadConfig() {
 }
 
 async function setConfigFromEnv() {
-    let config_items = getEnvConfigItems();
-    let success = config_api.setConfigItems(config_items);
+    const config_items = getEnvConfigItems();
+    if (!config_items || config_items.length === 0) return true;
+    const success = config_api.setConfigItems(config_items);
     if (success) {
         logger.info('Config items set using ENV variables.');
         await utils.wait(100);
@@ -500,13 +518,11 @@ async function loadConfig() {
     loadConfigValues();
 
     // connect to DB
-    await db_api.connectToDB();
+    if (!config_api.getConfigItem('ytdl_use_local_db'))
+        await db_api.connectToDB();
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
-    // creates archive path if missing
-    await fs.ensureDir(archivePath);
-
     // check migrations
     await checkMigrations();
 
@@ -552,14 +568,7 @@ function loadConfigValues() {
     url_domain = new URL(url);
 
     let logger_level = config_api.getConfigItem('ytdl_logger_level');
-    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
-    if (!possible_levels.includes(logger_level)) {
-        logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
-        logger_level = 'info';
-    }
-    logger.level = logger_level;
-    winston.loggers.get('console').level = logger_level;
-    logger.transports[2].level = logger_level;
+    utils.updateLoggerLevel(logger_level);
 }
 
 function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -575,7 +584,11 @@ async function watchSubscriptions() {
 
     if (!subscriptions) return;
 
-    const valid_subscriptions = subscriptions.filter(sub => !sub.paused);
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
 
     let subscriptions_amount = valid_subscriptions.length;
     let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
@@ -690,7 +703,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -763,7 +776,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
 
 app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
     const db_info = await db_api.getDBStats();
-    res.send({db_info: db_info});
+    res.send(db_info);
 });
 
 app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -803,11 +816,13 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings
+        cropFileSettings: req.body.cropFileSettings,
+        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -833,6 +848,7 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
@@ -911,31 +927,15 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
-    let files = null;
-    let playlists = null;
-    let sort = req.body.sort;
-    let range = req.body.range;
-    let text_search = req.body.text_search;
-    let file_type_filter = req.body.file_type_filter;
+    const sort = req.body.sort;
+    const range = req.body.range;
+    const text_search = req.body.text_search;
+    const file_type_filter = req.body.file_type_filter;
+    const favorite_filter = req.body.favorite_filter;
+    const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const filter_obj = {user_uid: uuid};
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
-    const file_count = await db_api.getRecords('files', filter_obj, true);
-
-    files = JSON.parse(JSON.stringify(files));
+    const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
 
     res.send({
         files: files,
@@ -1078,9 +1078,6 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
             await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
         } else if (is_playlist) {
             await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
-        } else if (type === 'subscription') {
-            // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
-            //          time they are requested from the subscription directory.
         } else {
             // error
             success = false;
@@ -1095,7 +1092,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
     });
 });
 
-app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
+app.post('/api/incrementViewCount', async (req, res) => {
     let file_uid = req.body.file_uid;
     let sub_id = req.body.sub_id;
     let uuid = req.body.uuid;
@@ -1230,12 +1227,9 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
-    let file = req.body.file;
     let file_uid = req.body.file_uid;
-    let sub = req.body.sub;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
+    let success = await db_api.deleteFile(file_uid, deleteForever);
 
     if (success) {
         res.send({
@@ -1268,7 +1262,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
     subscription = JSON.parse(JSON.stringify(subscription));
 
     // get sub videos
-    if (subscription.name && !subscription.streamingOnly) {
+    if (subscription.name) {
         var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
         subscription['videos'] = parsed_files;
         // loop through files for extra processing
@@ -1278,19 +1272,6 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
         }
 
-        res.send({
-            subscription: subscription,
-            files: parsed_files
-        });
-    } else if (subscription.name && subscription.streamingOnly) {
-        // return list of videos
-        let parsed_files = [];
-        if (subscription.videos) {
-            for (let i = 0; i < subscription.videos.length; i++) {
-                const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
-            }
-        }
         res.send({
             subscription: subscription,
             files: parsed_files
@@ -1335,9 +1316,8 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
 app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
-    let type = req.body.type;
 
-    const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1365,7 +1345,6 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     res.send({
         playlist: playlist,
         file_objs: file_objs,
-        type: playlist && playlist.type,
         success: !!playlist
     });
 });
@@ -1431,10 +1410,9 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
 app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const uid = req.body.uid;
     const blacklistMode = req.body.blacklistMode;
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     let wasDeleted = false;
-    wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
+    wasDeleted = await db_api.deleteFile(uid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1466,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
+        wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1527,20 +1505,69 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     });
 });
 
+app.post('/api/getArchives', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id;
+    const filter_obj = {user_uid: uuid, sub_id: sub_id};
+    const type = req.body.type;
+
+    // we do this for file types because if type is null, that means get files of all types
+    if (type) filter_obj['type'] = type;
+
+    const archives = await db_api.getRecords('archives', filter_obj);
+
+    res.send({
+        archives: archives
+    });
+});
+
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    let sub = req.body.sub;
-    let archive_dir = sub.archive;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
 
-    let full_archive_path = path.join(archive_dir, 'archive.txt');
+    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
 
-    if (await fs.pathExists(full_archive_path)) {
-        res.sendFile(full_archive_path);
+    if (archive_text !== null && archive_text !== undefined) {
+        res.setHeader('Content-type', "application/octet-stream");
+        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
+        res.send(archive_text);
     } else {
-        res.sendStatus(404);
+        res.sendStatus(400);
     }
 
 });
 
+app.post('/api/importArchive', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archive = req.body.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
+
+    const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
+
+    const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
+
+    res.send({
+        success: !!imported_count,
+        imported_count: imported_count
+    });
+});
+
+app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archives = req.body.archives;
+
+    let success = true;
+    for (const archive of archives) {
+        success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
+    }
+
+    res.send({
+        success: success
+    });
+});
+
 var upload_multer = multer({ dest: __dirname + '/appdata/' });
 app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
     const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1612,7 +1639,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
-        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
+        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1732,8 +1759,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
 
 app.post('/api/restartDownload', optionalJwt, async (req, res) => {
     const download_uid = req.body.download_uid;
-    const success = await downloader_api.restartDownload(download_uid);
-    res.send({success: success});
+    const new_download = await downloader_api.restartDownload(download_uid);
+    res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
 });
 
 app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1810,6 +1837,15 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
+    const task_key = req.body.task_key;
+    const new_options = req.body.new_options;
+  
+    const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
+
+    res.send({success: success});
+});
+
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
     fs.ensureDirSync(backup_dir);
@@ -1993,6 +2029,93 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+// notifications
+
+app.post('/api/getNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
+
+    res.send({notifications: notifications});
+});
+
+// set notifications to read
+app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
+    const uid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeRecord('notifications', {uid: uid});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
+
+    res.send({success: success});
+});
+
+// rss feed
+
+app.get('/api/rss', async function (req, res) {
+    if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
+        logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
+        res.sendStatus(403);
+        return;
+    }
+
+    // these are returned
+    const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
+    const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
+    const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
+    const file_type_filter = req.query.file_type_filter;
+    const favorite_filter = req.query.favorite_filter === 'true';
+    const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
+    const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
+
+    const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+
+    const feed = new Feed({
+            title: 'Downloads',
+            description: 'YoutubeDL-Material downloads',
+            id: utils.getBaseURL(),
+            link: utils.getBaseURL(),
+            image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
+            favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
+            generator: 'YoutubeDL-Material'
+    });
+
+    files.forEach(file => {
+        feed.addItem({
+            title: file.title,
+            link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
+            description: file.description,
+            author: [
+                {
+                    name: file.uploader,
+                    link: file.url
+                }
+            ],
+            contributor: [],
+            date: file.timestamp,
+            // https://stackoverflow.com/a/45415677/8088021
+            image: file.thumbnailURL.replace('&', '&amp;')
+        });
+      });
+    res.send(feed.rss2());
+});
+
+// web server
+
 app.use(function(req, res, next) {
     //if the request is not html then move along
     var accept = req.accepts('html', 'json', 'xml');

backend/appdata/default.json

@@ -23,7 +23,12 @@
       "download_only_mode": false,
       "allow_autoplay": true,
       "enable_downloads_manager": true,
-      "allow_playlist_categorization": true
+      "allow_playlist_categorization": true,
+      "force_autoplay": false,
+      "enable_notifications": true,
+      "enable_all_notifications": true,
+      "allowed_notification_types": [],
+      "enable_rss_feed": false
     },
     "API": {
       "use_API_key": false,
@@ -31,10 +36,21 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
-      "generate_NFO_files": false
+      "generate_NFO_files": false,
+      "use_ntfy_API": false,
+      "ntfy_topic_URL": "",
+      "use_gotify_API": false,
+      "gotify_server_URL": "",
+      "gotify_app_token": "",
+      "use_telegram_API": false,
+      "telegram_bot_token": "",
+      "telegram_chat_id": "",
+      "webhook_URL": "",
+      "discord_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",
@@ -63,7 +79,7 @@
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
-      "default_downloader": "youtube-dl",
+      "default_downloader": "yt-dlp",
       "use_default_downloading_agent": true,
       "custom_downloading_agent": "",
       "multi_user_mode": false,

backend/archive.js

@@ -0,0 +1,91 @@
+const path = require('path');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+
+const db_api = require('./db');
+
+exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
+    const filter = {user_uid: user_uid, sub_id: sub_id};
+    if (type) filter['type'] = type;
+    const archive_items = await db_api.getRecords('archives', filter);
+    const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
+    return archive_item_lines.join('\n');
+}
+
+exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
+    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
+    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
+    return success;
+}
+
+exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return success;
+}
+
+exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
+    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return !!archive_item;
+}
+
+exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
+    let archive_import_count = 0;
+    const lines = archive_text.split('\n');
+    for (let line of lines) {
+        const archive_line_parts = line.trim().split(' ');
+        // should just be the extractor and the video ID
+        if (archive_line_parts.length !== 2) {
+            continue;
+        }
+
+        const extractor = archive_line_parts[0];
+        const id        = archive_line_parts[1];
+        if (!extractor || !id) continue;
+
+        // we can't do a bulk write because we need to avoid duplicate archive items existing in db
+
+        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
+        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
+        archive_import_count++;
+    }
+    return archive_import_count;
+}
+
+exports.importArchives = async () => {
+    const imported_archives = [];
+    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        if (!dir_to_check['archive_path']) continue;
+
+        const files_to_import = [
+            path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
+            path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
+        ]
+
+        for (const file_to_import of files_to_import) {
+            const file_exists = await fs.pathExists(file_to_import);
+            if (!file_exists) continue;
+
+            const archive_text = await fs.readFile(file_to_import, 'utf8');
+            await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
+            imported_archives.push(file_to_import);
+        }
+    }
+    return imported_archives;
+}
+
+const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
+    return {
+        extractor: extractor,
+        id: id,
+        type: type,
+        title: title,
+        user_uid: user_uid ? user_uid : null,
+        sub_id: sub_id ? sub_id : null,
+        timestamp: Date.now() / 1000,
+        uid: uuid()
+    }
+}

backend/authentication/auth.js

@@ -33,7 +33,14 @@ exports.initialize = function () {
 
   saltRounds = 10;
 
+  // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
+  if (!(+JWT_EXPIRATION)) {
+    logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
+    JWT_EXPIRATION = 86400;
+  } else {
+    JWT_EXPIRATION = +JWT_EXPIRATION;
+  }
 
   SERVER_SECRET = null;
   if (db_api.users_db.get('jwt_secret').value()) {
@@ -361,7 +368,6 @@ exports.userHasPermission = async function(user_uid, permission) {
     logger.error('Invalid role ' + role);
     return false;
   }
-  const role_permissions = (await db_api.getRecords('roles'))['permissions'];
 
   const user_has_explicit_permission = user_obj['permissions'].includes(permission);
   const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -376,7 +382,8 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 
   // no overrides, let's check if the role has the permission
-  if (role_permissions.includes(permission)) {
+  const role_has_permission = await exports.roleHasPermissions(role, permission);
+  if (role_has_permission) {
     return true;
   } else {
     logger.verbose(`User ${user_uid} failed to get permission ${permission}`);
@@ -384,6 +391,16 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 }
 
+exports.roleHasPermissions = async function(role, permission) {
+  const role_obj = await db_api.getRecord('roles', {key: role})
+  if (!role) {
+    logger.error(`Role ${role} does not exist!`);
+  }
+  const role_permissions = role_obj['permissions'];
+  if (role_permissions && role_permissions.includes(permission)) return true;
+  else return false;
+}
+
 exports.userPermissions = async function(user_uid) {
   let user_permissions = [];
   const user_obj = await db_api.getRecord('users', ({uid: user_uid}));

backend/config.js

@@ -185,7 +185,6 @@ const DEFAULT_CONFIG = {
         "default_file_output": "",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false,
         "include_thumbnail": true,
         "include_metadata": true,
         "max_concurrent_downloads": 5,
@@ -196,20 +195,32 @@ const DEFAULT_CONFIG = {
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_autoplay": true,
+        "force_autoplay": false,
         "enable_downloads_manager": true,
-        "allow_playlist_categorization": true
+        "allow_playlist_categorization": true,
+        "enable_notifications": true,
+        "enable_all_notifications": true,
+        "allowed_notification_types": [],
+        "enable_rss_feed": false,
       },
       "API": {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
-        "use_twitch_API": false,
-        "twitch_API_key": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
-        "generate_NFO_files": false
+        "generate_NFO_files": false,
+        "use_ntfy_API": false,
+        "ntfy_topic_URL": "",
+        "use_gotify_API": false,
+        "gotify_server_URL": "",
+        "gotify_app_token": "",
+        "use_telegram_API": false,
+        "telegram_bot_token": "",
+        "telegram_chat_id": "",
+        "webhook_URL": "",
+        "discord_webhook_URL": ""
       },
       "Themes": {
         "default_theme": "default",
@@ -238,7 +249,7 @@ const DEFAULT_CONFIG = {
         "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
       },
       "Advanced": {
-        "default_downloader": "youtube-dl",
+        "default_downloader": "yt-dlp",
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",
         "multi_user_mode": false,

backend/consts.js

@@ -30,10 +30,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'
     },
-    'ytdl_safe_download_override': {
-        'key': 'ytdl_safe_download_override',
-        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
-    },
     'ytdl_include_thumbnail': {
         'key': 'ytdl_include_thumbnail',
         'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -68,9 +64,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_autoplay': {
-        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+    'ytdl_force_autoplay': {
+        'key': 'ytdl_force_autoplay',
+        'path': 'YoutubeDLMaterial.Extra.force_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -80,6 +76,22 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_allow_playlist_categorization',
         'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
+    'ytdl_enable_notifications': {
+        'key': 'ytdl_enable_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_notifications'
+    },
+    'ytdl_enable_all_notifications': {
+        'key': 'ytdl_enable_all_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
+    },
+    'ytdl_allowed_notification_types': {
+        'key': 'ytdl_allowed_notification_types',
+        'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
+    },
+    'ytdl_enable_rss_feed': {
+        'key': 'ytdl_enable_rss_feed',
+        'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
+    },
 
     // API
     'ytdl_use_api_key': {
@@ -98,14 +110,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
-    'ytdl_use_twitch_api': {
-        'key': 'ytdl_use_twitch_api',
-        'path': 'YoutubeDLMaterial.API.use_twitch_API'
-    },
-    'ytdl_twitch_api_key': {
-        'key': 'ytdl_twitch_api_key',
-        'path': 'YoutubeDLMaterial.API.twitch_API_key'
-    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -118,6 +122,46 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_generate_nfo_files',
         'path': 'YoutubeDLMaterial.API.generate_NFO_files'
     },
+    'ytdl_use_ntfy_API': {
+        'key': 'ytdl_use_ntfy_API',
+        'path': 'YoutubeDLMaterial.API.use_ntfy_API'
+    },
+    'ytdl_ntfy_topic_url': {
+        'key': 'ytdl_ntfy_topic_url',
+        'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
+    },
+    'ytdl_use_gotify_API': {
+        'key': 'ytdl_use_gotify_API',
+        'path': 'YoutubeDLMaterial.API.use_gotify_API'
+    },
+    'ytdl_gotify_server_url': {
+        'key': 'ytdl_gotify_server_url',
+        'path': 'YoutubeDLMaterial.API.gotify_server_URL'
+    },
+    'ytdl_gotify_app_token': {
+        'key': 'ytdl_gotify_app_token',
+        'path': 'YoutubeDLMaterial.API.gotify_app_token'
+    },
+    'ytdl_use_telegram_API': {
+        'key': 'ytdl_use_telegram_API',
+        'path': 'YoutubeDLMaterial.API.use_telegram_API'
+    },
+    'ytdl_telegram_bot_token': {
+        'key': 'ytdl_telegram_bot_token',
+        'path': 'YoutubeDLMaterial.API.telegram_bot_token'
+    },
+    'ytdl_telegram_chat_id': {
+        'key': 'ytdl_telegram_chat_id',
+        'path': 'YoutubeDLMaterial.API.telegram_chat_id'
+    },
+    'ytdl_webhook_url': {
+        'key': 'ytdl_webhook_url',
+        'path': 'YoutubeDLMaterial.API.webhook_URL'
+    },
+    'ytdl_discord_webhook_url': {
+        'key': 'ytdl_discord_webhook_url',
+        'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
+    },
 
 
     // Themes
@@ -217,7 +261,8 @@ exports.AVAILABLE_PERMISSIONS = [
     'subscriptions',
     'sharing',
     'advanced_download',
-    'downloads_manager'
+    'downloads_manager',
+    'tasks_manager'
 ];
 
 exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
@@ -301,4 +346,6 @@ const YTDL_ARGS_WITH_VALUES = [
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
-exports.CURRENT_VERSION = 'v4.2';
+exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
+
+exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -2,6 +2,7 @@ var fs = require('fs-extra')
 var path = require('path')
 const { MongoClient } = require("mongodb");
 const { uuid } = require('uuidv4');
+const _ = require('lodash');
 
 const config_api = require('./config');
 var utils = require('./utils')
@@ -58,6 +59,13 @@ const tables = {
         name: 'tasks',
         primary_key: 'key'
     },
+    notifications: {
+        name: 'notifications',
+        primary_key: 'uid'
+    },
+    archives: {
+        name: 'archives'
+    },
     test: {
         name: 'test'
     }
@@ -148,6 +156,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
                 await database.collection(table).createIndex(text_search);
             }
         });
+        using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
         return true;
     } catch(err) {
         logger.error(err);
@@ -198,7 +207,7 @@ async function registerFileDBManual(file_object) {
     path_object = path.parse(file_object['path']);
     file_object['path'] = path.format(path_object);
 
-    exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+    await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
 
     return file_object;
 }
@@ -252,13 +261,16 @@ exports.getFileDirectoriesAndDBs = async () => {
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'audio'),
                 user_uid: user.uid,
-                type: 'audio'
+                type: 'audio',
+                archive_path: utils.getArchiveFolder('audio', user.uid)
             });
 
             // add user's video dir to check list
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'video'),
-                type: 'video'
+                user_uid: user.uid,
+                type: 'video',
+                archive_path: utils.getArchiveFolder('video', user.uid)
             });
         }
     } else {
@@ -268,13 +280,15 @@ exports.getFileDirectoriesAndDBs = async () => {
         // add audio dir to check list
         dirs_to_check.push({
             basePath: audioFolderPath,
-            type: 'audio'
+            type: 'audio',
+            archive_path: utils.getArchiveFolder('audio')
         });
 
         // add video dir to check list
         dirs_to_check.push({
             basePath: videoFolderPath,
-            type: 'video'
+            type: 'video',
+            archive_path: utils.getArchiveFolder('video')
         });
     }
 
@@ -295,7 +309,8 @@ exports.getFileDirectoriesAndDBs = async () => {
                                       : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
             user_uid: subscription_to_check.user_uid,
             type: subscription_to_check.type,
-            sub_id: subscription_to_check['id']
+            sub_id: subscription_to_check['id'],
+            archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
         });
     }
 
@@ -350,14 +365,14 @@ exports.addMetadataPropertyToDB = async (property_key) => {
             }
         }
 
-        return await exports.bulkUpdateRecords('files', 'uid', update_obj);
+        return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
     } catch(err) {
         logger.error(err);
         return false;
     }
 }
 
-exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
     const first_video = await exports.getVideo(uids[0]);
     const thumbnailToUse = first_video['thumbnailURL'];
     
@@ -366,7 +381,6 @@ exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
         uids: uids,
         id: uuid(),
         thumbnailURL: thumbnailToUse,
-        type: type,
         registered: Date.now(),
         randomize_order: false
     };
@@ -448,8 +462,8 @@ exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null)
     return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
 }
 
-exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid, uuid);
+exports.deleteFile = async (uid, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid);
     const type = file_obj.isAudio ? 'audio' : 'video';
     const folderPath = path.dirname(file_obj.path);
     const ext = type === 'audio' ? 'mp3' : 'mp4';
@@ -495,22 +509,21 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
+        // get id/extractor from JSON
 
-        // get ID from JSON
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedID = null;
+        let retrievedExtractor = null;
+        if (info_json) {
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
 
-        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let id = null;
-        if (jsonobj) id = jsonobj.id;
-
-        // use subscriptions API to remove video from the archive file, and write it to the blacklist
-        if (await fs.pathExists(archive_path)) {
-            const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
-            if (blacklistMode && line) await writeToBlacklist(type, line);
-        } else {
-            logger.info('Could not find archive file for audio files. Creating...');
-            await fs.close(await fs.open(archive_path, 'w'));
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        if (!blacklistMode) {
+            // workaround until a files_api is created (using archive_api would make a circular dependency)
+            await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
+            // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
         }
     }
 
@@ -542,8 +555,32 @@ exports.getVideo = async (file_uid) => {
     return await exports.getRecord('files', {uid: file_uid});
 }
 
-exports.getFiles = async (uuid = null) => {
-    return await exports.getRecords('files', {user_uid: uuid});
+exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (favorite_filter) {
+        filter_obj['favorite'] = true;
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
+    const file_count = await exports.getRecords('files', filter_obj, true);
+
+    return {files, file_count};
 }
 
 exports.setVideoProperty = async (file_uid, assignment_obj) => {
@@ -558,7 +595,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
 exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
     // local db override
     if (using_local_db) {
-        if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
+        if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
         local_db.get(table).push(doc).write();
         return true;
     }
@@ -661,9 +698,15 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 
 // Update
 
-exports.updateRecord = async (table, filter_obj, update_obj) => {
+exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
     // local db override
     if (using_local_db) {
+        if (nested_mode) {
+            // if object is nested we need to handle it differently
+            update_obj = utils.convertFlatObjectToNestedObject(update_obj);
+            exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
+            return true;
+        }
         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
@@ -677,7 +720,14 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
+            const props_to_update = Object.keys(update_obj);
+            for (let i = 0; i < props_to_update.length; i++) {
+                const prop_to_update = props_to_update[i];
+                const prop_value = update_obj[prop_to_update];
+                record[prop_to_update] = prop_value;
+            }
+        }).write();
         return true;
     }
 
@@ -685,7 +735,19 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
     return !!(output['result']['ok']);
 }
 
-exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
+exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
+    // local db override
+    if (using_local_db) {
+        const props_to_remove = Object.keys(remove_obj);
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
+        return true;
+    }
+
+    const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
+    return !!(output['result']['ok']);
+}
+
+exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
     // local db override
     if (using_local_db) {
         local_db.get(table).each((record) => {
@@ -933,6 +995,7 @@ exports.importJSONToDB = async (db_json, users_json) => {
 const createFilesRecords = (files, subscriptions) => {
     for (let i = 0; i < subscriptions.length; i++) {
         const subscription = subscriptions[i];
+        if (!subscription['videos']) continue;
         subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
         files = files.concat(subscriptions[i]['videos']);
     }
@@ -993,7 +1056,7 @@ exports.backupDB = async () => {
     const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
     const path_to_backups = path.join(backup_dir, backup_file_name);
 
-    logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
+    logger.info(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
 
     const table_to_records = {};
     for (let i = 0; i < tables_list.length; i++) {
@@ -1040,10 +1103,11 @@ exports.transferDB = async (local_to_remote) => {
         table_to_records[table] = await exports.getRecords(table);
     }
 
+    logger.info('Backup up DB...');
+    await exports.backupDB(); // should backup always
+
     using_local_db = !local_to_remote;
     if (local_to_remote) {
-        logger.debug('Backup up DB...');
-        await exports.backupDB();
         const db_connected = await exports.connectToDB(5, true);
         if (!db_connected) {
             logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1097,6 +1161,14 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
                     } else if ('$ne' in filter_prop_value) {
                         filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
+                    } else if ('$lt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
+                    } else if ('$gt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
+                    } else if ('$lte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
+                    } else if ('$gte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
                     }
                 } else {
                     // handle case of nested property check
@@ -1112,14 +1184,7 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     return return_val;
 }
 
-// archive helper functions
-
-async function writeToBlacklist(type, line) {
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}
+// should only be used for tests
+exports.setLocalDBMode = (mode) => {
+    using_local_db = mode;
+}

backend/downloader.js

@@ -1,7 +1,6 @@
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const path = require('path');
-const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
@@ -14,12 +13,12 @@ const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
+const notifications_api = require('./notifications');
+const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
 if (db_api.database_initialized) {
     setupDownloads();
 } else {
@@ -28,7 +27,26 @@ if (db_api.database_initialized) {
     });
 }
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+/*
+
+This file handles all the downloading functionality.
+
+To download a file, we go through 4 steps. Here they are with their respective index & function:
+
+0: Create the download
+ - createDownload()
+1: Get info for the download (we need this step for categories and archive functionality)
+ - collectInfo()
+2: Download the file
+ - downloadQueuedFile()
+3: Complete
+ - N/A
+
+We use checkDownloads() to move downloads through the steps and call their respective functions.
+
+*/
+
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -37,6 +55,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             user_uid: user_uid,
             sub_id: sub_id,
             sub_name: sub_name,
+            prefetched_info: prefetched_info,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -85,10 +104,10 @@ exports.resumeDownload = async (download_uid) => {
 exports.restartDownload = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     await exports.clearDownload(download_uid);
-    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
     
     should_check_downloads = true;
-    return success;
+    return new_download;
 }
 
 exports.cancelDownload = async (download_uid) => {
@@ -107,9 +126,10 @@ exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download_uid, error_message) {
-    if (!download_uid) return;
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+async function handleDownloadError(download, error_message, error_type = null) {
+    if (!download || !download['uid']) return;
+    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
+    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
 async function setupDownloads() {
@@ -155,6 +175,13 @@ async function checkDownloads() {
         if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
 
         if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            if (waiting_download['sub_id']) {
+                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
+                if (sub_missing) {
+                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    continue;
+                }
+            }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
@@ -187,13 +214,27 @@ async function collectInfo(download_uid) {
     let args = await exports.generateArgs(url, type, options, download['user_uid']);
 
     // get video info prior to download
-    let info = await exports.getVideoInfoByURL(url, args, download_uid);
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
     if (!info) {
         // info failed, error presumably already recorded
         return;
     }
 
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
+        if (exists_in_archive) {
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            logger.warn(error);
+            if (download_uid) {
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error, 'exists_in_archive');
+                return;
+            }
+        }
+    }
+
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -207,7 +248,7 @@ async function collectInfo(download_uid) {
         info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
-    download['category'] = category;
+    const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
 
     // setup info required to calculate download progress
 
@@ -229,7 +270,9 @@ async function collectInfo(download_uid) {
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    category: stripped_category,
+                                                                    prefetched_info: null
                                                                 });
 }
 
@@ -242,6 +285,7 @@ async function downloadQueuedFile(download_uid) {
     return new Promise(async resolve => {
         const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
         await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
 
         const url = download['url'];
@@ -249,9 +293,11 @@ async function downloadQueuedFile(download_uid) {
         const options = download['options'];
         const args = download['args'];
         const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
         if (options.customFileFolderPath) {
             fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
         }
         fs.ensureDirSync(fileFolderPath);
 
@@ -268,14 +314,14 @@ async function downloadQueuedFile(download_uid) {
             clearInterval(download_checker);
             if (err) {
                 logger.error(err.stderr);
-                await handleDownloadError(download_uid, err.stderr);
+                await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_output');
                     logger.warn(error_message);
                     resolve(false);
                     return;
@@ -284,7 +330,10 @@ async function downloadQueuedFile(download_uid) {
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
-                        output_json = JSON.parse(output[i]);
+                        // we have to do this because sometimes there will be leading characters before the actual json
+                        const start_idx = output[i].indexOf('{"');
+                        const clean_output = output[i].slice(start_idx, output[i].length);
+                        output_json = JSON.parse(clean_output);
                     } catch(e) {
                         output_json = null;
                     }
@@ -301,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -337,15 +386,12 @@ async function downloadQueuedFile(download_uid) {
                     // registers file in DB
                     const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
-                    file_objs.push(file_obj);
-                }
+                    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
 
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
+                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
+
+                    file_objs.push(file_obj);
                 }
 
                 let container = null;
@@ -353,13 +399,13 @@ async function downloadQueuedFile(download_uid) {
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
                 } else {
                     const error_message = 'Downloaded file failed to result in metadata object.';
                     logger.error(error_message);
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_metadata');
                 }
 
                 const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -373,15 +419,27 @@ async function downloadQueuedFile(download_uid) {
 // helper functions
 
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+
+    if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
+        logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
+    }
+
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
 
     const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
     const globalArgs = config_api.getConfigItem('ytdl_custom_args');
     const useCookies = config_api.getConfigItem('ytdl_use_cookies');
     const is_audio = type === 'audio';
 
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
 
     if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
 
@@ -391,6 +449,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 
     // video-specific args
     const selectedHeight = options.selectedHeight;
+    const maxHeight = options.maxHeight;
+    const heightParam = selectedHeight || maxHeight;
 
     // audio-specific args
     const maxBitrate = options.maxBitrate;
@@ -404,8 +464,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     if (!is_audio && !is_youtube) {
         // tiktok videos fail when using the default format
         qualityPath = null;
-    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
-        qualityPath = ['-f', 'bestvideo+bestaudio']
     }
 
     if (customArgs) {
@@ -413,8 +471,9 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     } else {
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
-        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+        } else if (heightParam && heightParam !== '' && !is_audio) {
+            const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
+            qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -451,28 +510,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -496,9 +533,11 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.push('-r', rate_limit);
         }
         
-        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
         if (default_downloader === 'yt-dlp') {
-            downloadConfig.push('--no-clean-infojson');
+            downloadConfig = utils.filterArgs(downloadConfig, ['--print-json']);
+
+            // in yt-dlp -j --no-simulate is preferable
+            downloadConfig.push('--no-clean-info-json', '-j', '--no-simulate');
         }
 
     }
@@ -506,14 +545,15 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     // filter out incompatible args
     downloadConfig = filterArgs(downloadConfig, is_audio);
 
-    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
     return downloadConfig;
 }
 
 exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
-        const new_args = [...args];
+        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...temp_args];
 
         const archiveArgIndex = new_args.indexOf('--download-archive');
         if (archiveArgIndex !== -1) {
@@ -545,7 +585,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                     const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
                     logger.error(error);
                     if (download_uid) {
-                        await handleDownloadError(download_uid, error);
+                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download, error, 'parse_failed');
                     }
                     resolve(null);
                 }
@@ -554,7 +595,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                 if (err.stderr) error_message += `\n\n${err.stderr}`;
                 logger.error(error_message);
                 if (download_uid) {
-                    await handleDownloadError(download_uid, error_message);
+                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
                 }
                 resolve(null);
             }
@@ -565,8 +607,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
 function filterArgs(args, isAudio) {
     const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
     const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
-    return args.filter(x => !args_to_remove.includes(x));
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
 }
 
 async function checkDownloadPercent(download_uid) {
@@ -621,13 +662,3 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join(archivePath);
-    }
-}

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="pm2-runtime pm2.config.js"
+CMD="npm start"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then
@@ -10,7 +10,7 @@ fi
 
 # chown current working directory to current user
 if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
   exec gosu "$UID:$GID" "$0" "$@"
 fi
 

backend/fix-scripts/001-fix_download_permissions.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M
 # Date: 2022-05-03
@@ -6,8 +6,7 @@
 # If you want to run this script on a bare-metal installation instead of within Docker
 # make sure that the paths configured below match your paths! (it's wise to use the full paths)
 # USAGE: within your container's bash shell:
-# chmod -R +x ./fix-scripts/
-# ./fix-scripts/001-fix_download_permissions.sh
+# ./fix-scripts/<name of fix-script>
 
 # User defines / Docker env defaults
 PATH_SUBS=/app/subscriptions

backend/fix-scripts/002-fix_dupes_per_archive_file.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-09
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines (NO TRAILING SLASHES) / Docker env defaults
+PATH_SUBSARCHIVE=/app/subscriptions/archives
+PATH_ONEOFFARCHIVE=/app/appdata/archives
+
+# Backup paths (substitute with your personal preference if you like)
+PATH_SUBSARCHIVEBKP=$PATH_SUBSARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+PATH_ONEOFFARCHIVEBKP=$PATH_ONEOFFARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+
+
+# Define Colors for TUI
+yellow=$(tput setaf 3)
+normal=$(tput sgr0)
+
+tput civis                                                          # hide the cursor
+
+clear -x
+printf "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "Welcome to the INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M."
+printf "\nThis script will cycle through the archive files in the folders mentioned"
+printf "\nbelow and remove within each archive the dupe entries. (compact them)"
+printf "\nDuring some older builds of YTDL-M the archives could receive dupe"
+printf "\nentries and blow up in size, sometimes causing conflicts with download management."
+printf '\n%*s' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBSARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found Subscriptions archive directory at ${PATH_SUBSARCHIVE}"
+[ -d $PATH_ONEOFFARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found one-off archive directory at ${PATH_ONEOFFARCHIVE}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    printf "\n\n  Couldn't find any archive location path! \n\nPlease edit this script to configure!"
+    tput cnorm
+    exit 2;;
+  2)
+    printf "\n\n  Found all archive locations. \n\nProceed? (Y/N)";;
+  *)
+    printf "\n\n  Only found ${i} out of 2 archive locations! Something about this script's config must be wrong. \n\nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg          # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    printf "\n\nRunning jobs now... (this may take a while)\n"
+
+    printf "\nBacking up directories...\n"
+
+    chars="⣾⣽⣻⢿⡿⣟⣯⣷"
+    cp -R $PATH_SUBSARCHIVE $PATH_SUBSARCHIVEBKP &
+    PID=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.15
+    done
+    [ -d $PATH_SUBSARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_SUBSARCHIVE} to ${PATH_SUBSARCHIVEBKP} ($(du -sh $PATH_SUBSARCHIVEBKP | cut -f1))\n"
+
+    cp -R $PATH_ONEOFFARCHIVE $PATH_ONEOFFARCHIVEBKP &
+    PID2=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID2 ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.1
+    done
+    [ -d $PATH_ONEOFFARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_ONEOFFARCHIVE} to ${PATH_ONEOFFARCHIVEBKP} ($(du -sh $PATH_ONEOFFARCHIVEBKP | cut -f1))\n"
+
+
+    printf "\nCompacting files...\n"
+
+    tmpfile=$(mktemp) &&
+
+    [ -d $PATH_SUBSARCHIVE ] &&
+    find $PATH_SUBSARCHIVE -name '*.txt' -print0 | while read -d $'\0' file # Set delimiter to null because we want to catch all possible filenames (WE CANNOT CHANGE IFS HERE) - https://stackoverflow.com/a/15931055
+    do
+        cp "$file" "$tmpfile"
+        { awk '!x[$0]++' "$tmpfile" > "$file"; } &                         # https://unix.stackexchange.com/questions/159695/how-does-awk-a0-work
+        PID3=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID3 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [[ "$AFTER" -ne "$BEFORE" ]]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action needed for file: ${file}\n"
+        fi
+    done
+
+    [ -d $PATH_ONEOFFARCHIVE ] &&
+    find $PATH_ONEOFFARCHIVE -name '*.txt' -print0 | while read -d $'\0' file
+    do
+        cp "$file" "$tmpfile" &
+        awk '!x[$0]++' "$tmpfile" > "$file" &
+        PID4=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID4 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [ "$BEFORE" -ne "$AFTER" ]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action ran for file: ${file}\n"
+        fi
+    done
+    tput cnorm                                                     # show the cursor
+    rm "$tmpfile"
+
+    printf "\n\n✔ Done."
+    printf "\nℹ Please keep in mind that you may still want to"
+    printf "\n  run corruption checks against your archives!\n\n"
+    exit
+else
+    tput cnorm
+    printf "\nOkay, bye.\n\n"
+    exit
+fi

backend/notifications.js

@@ -0,0 +1,187 @@
+const db_api = require('./db');
+const config_api = require('./config');
+const logger = require('./logger');
+const utils = require('./utils');
+const consts = require('./consts');
+
+const { uuid } = require('uuidv4');
+
+const fetch = require('node-fetch');
+const { gotify } = require("gotify");
+const TelegramBot = require('node-telegram-bot-api');
+const REST = require('@discordjs/rest').REST;
+const API = require('@discordjs/core').API;
+const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
+
+const NOTIFICATION_TYPE_TO_TITLE = {
+    task_finished: 'Task finished',
+    download_complete: 'Download complete',
+    download_error: 'Download error'
+}
+
+const NOTIFICATION_TYPE_TO_BODY = {
+    task_finished: (notification) => notification['data']['task_title'],
+    download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
+    download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
+}
+
+const NOTIFICATION_TYPE_TO_URL = {
+    task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
+    download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
+    download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
+}
+
+const NOTIFICATION_TYPE_TO_THUMBNAIL = {
+    task_finished: () => null,
+    download_complete: (notification) => notification['data']['file_thumbnail'],
+    download_error: () => null
+}
+
+exports.sendNotification = async (notification) => {
+    // info necessary if we are using 3rd party APIs
+    const type = notification['type'];
+
+    const data = {
+        title: NOTIFICATION_TYPE_TO_TITLE[type],
+        body: NOTIFICATION_TYPE_TO_BODY[type](notification),
+        type: type,
+        url: NOTIFICATION_TYPE_TO_URL[type](notification),
+        thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
+    }
+
+    if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
+        sendNtfyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
+        sendGotifyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
+        sendTelegramNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_webhook_url')) {
+        sendGenericNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
+        sendDiscordNotification(data);
+    }
+
+    await db_api.insertRecordIntoTable('notifications', notification);
+    return notification;
+}
+
+exports.sendTaskNotification = async (task_obj, confirmed) => {
+    if (!notificationEnabled('task_finished')) return;
+    // workaround for tasks which are user_uid agnostic
+    const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
+    await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
+    const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
+    const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadNotification = async (file, user_uid) => {
+    if (!notificationEnabled('download_complete')) return;
+    const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
+    const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
+    if (!notificationEnabled('download_error')) return;
+    const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
+    const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.createNotification = (type, actions, data, user_uid) => {
+    const notification = {
+        type: type,
+        actions: actions,
+        data: data,
+        user_uid: user_uid,
+        uid: uuid(),
+        read: false,
+        timestamp: Date.now()/1000
+    }
+    return notification;
+}
+
+function notificationEnabled(type) {
+    return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
+}
+
+function sendNtfyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to ntfy');
+    fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
+        method: 'POST',
+        body: body,
+        headers: {
+            'Title': title,
+            'Tags': type,
+            'Click': url,
+            'Attach': thumbnail
+        }
+    });
+}
+
+async function sendGotifyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to gotify');
+    await gotify({
+        server: config_api.getConfigItem('ytdl_gotify_server_url'),
+        app: config_api.getConfigItem('ytdl_gotify_app_token'),
+        title: title,
+        message: body,
+        tag: type,
+        priority: 5, // Keeping default from docs, may want to change this,
+        extras: {
+            "client::notification": {
+                click: { url: url },
+                bigImageUrl: thumbnail
+            }
+        }
+      });
+}
+
+async function sendTelegramNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to Telegram');
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    const bot = new TelegramBot(bot_token);
+    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+}
+
+async function sendDiscordNotification({body, title, type, url, thumbnail}) {
+    const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
+    const url_split = discord_webhook_url.split('webhooks/');
+    const [webhook_id, webhook_token] = url_split[1].split('/');
+    const rest = new REST({ version: '10' });
+    const api = new API(rest);
+    const embed = new EmbedBuilder()
+        .setTitle(title)
+        .setColor(0x00FFFF)
+        .setURL(url)
+        .setDescription(`ID: ${type}`);
+    if (thumbnail) embed.setThumbnail(thumbnail);
+    if (type === 'download_error') embed.setColor(0xFC2003);
+
+    const result = await api.webhooks.execute(webhook_id, webhook_token, {
+        content: body,
+        username: 'YoutubeDL-Material',
+        avatar_url: consts.ICON_URL,
+        embeds: [embed],
+    });
+    return result;
+}
+
+function sendGenericNotification(data) {
+    const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
+    logger.verbose(`Sending generic notification to ${webhook_url}`);
+    fetch(webhook_url, {
+        method: 'POST',
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify(data),
+    });
+}

backend/package.json

@@ -5,20 +5,9 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon app.js",
+    "start": "pm2-runtime --raw pm2.config.js",
     "debug": "set YTDL_MODE=debug && node app.js"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "*.js",
-      "appdata/*",
-      "public/*"
-    ],
-    "watch": [
-      "restart_update.json",
-      "restart_general.json"
-    ]
-  },
   "repository": {
     "type": "git",
     "url": ""
@@ -30,31 +19,36 @@
   },
   "homepage": "",
   "dependencies": {
+    "@discordjs/builders": "^1.6.1",
+    "@discordjs/core": "^0.5.2",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
-    "async-mutex": "^0.3.1",
+    "async-mutex": "^0.4.0",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
+    "express-session": "^1.17.3",
+    "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "gotify": "^1.1.0",
     "jsonwebtoken": "^8.5.1",
+    "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
-    "merge-files": "^0.1.2",
     "mocha": "^9.2.2",
-    "moment": "^2.29.2",
+    "moment": "^2.29.4",
     "mongodb": "^3.6.9",
-    "multer": "^1.4.2",
+    "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
-    "nodemon": "^2.0.7",
+    "node-telegram-bot-api": "^0.61.0",
     "passport": "^0.4.1",
     "passport-http": "^0.3.0",
-    "passport-jwt": "^4.0.0",
+    "passport-jwt": "^4.0.1",
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
@@ -63,7 +57,7 @@
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.0.6",
+    "uuidv4": "^6.2.13",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"

backend/pm2.config.js

@@ -6,4 +6,4 @@ module.exports = {
       out_file: "/dev/null",
       error_file: "/dev/null"
     }]
-}
+}

backend/subscriptions.js

@@ -3,6 +3,7 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
+const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
 
@@ -91,7 +92,10 @@ async function getSubscriptionInfo(sub) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
+                            let sub_name = sub.name;
+                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
+                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
                         }
                     }
 
@@ -138,28 +142,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
 
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     if (deleteMode && (await fs.pathExists(appendedBasePath))) {
-        if (sub.archive && (await fs.pathExists(sub.archive))) {
-            const archive_file_path = path.join(sub.archive, 'archive.txt');
-            // deletes archive if it exists
-            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
-            if (await fs.pathExists(archive_file_path)) {
-                await fs.unlink(archive_file_path);
-            }
-            await fs.rmdir(sub.archive);
-        }
         await fs.remove(appendedBasePath);
     }
+
+    await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
 async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    if (typeof sub === 'string') {
+        // TODO: fix bad workaround where sub is a sub_id
+        sub = await db_api.getRecord('subscriptions', {sub_id: sub});
+    }
     // TODO: combine this with deletefile
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
+    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -178,7 +179,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
+        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = info_json['id'];
+        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -196,12 +199,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (!deleteForever && useArchive && sub.archive && retrievedID) {
-                const archive_path = path.join(sub.archive, 'archive.txt')
-                // if archive exists, remove line with video ID
-                if (await fs.pathExists(archive_path)) {
-                    utils.removeIDFromArchive(archive_path, retrievedID);
-                }
+            const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useArchive && !deleteForever) {
+                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
         }
@@ -232,74 +232,39 @@ async function getVideosForSub(sub, user_uid = null) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable')) {
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        // TODO: reimplement
-
-                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
-                        // for (let i = 0; i < outputs.length; i++) {
-                        //     const output = JSON.parse(outputs[i]);
-                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                        //         // we found a video that errored! add it to the archive to prevent future errors
-                        //         if (sub.archive) {
-                        //             archive_dir = sub.archive;
-                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                        //             fs.appendFileSync(archive_path, output['id']);
-                        //         }
-                        //     }
-                        // }
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
+                        resolve(files_to_download);
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
                     }
+                } else {
+                    logger.error('Subscription check failed!');
                 }
                 resolve(false);
             } else if (output) {
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
-                    logger.verbose('No additional videos to download for ' + sub.name);
-                    resolve(true);
-                    return;
-                }
-
-                const output_jsons = [];
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                        output_jsons.push(output_json);
-                    } catch(e) {
-                        output_json = null;
-                    }
-                    if (!output_json) {
-                        continue;
-                    }
-                }
-
-                const files_to_download = await getFilesToDownload(sub, output_jsons);
-                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
-
-                for (let j = 0; j < files_to_download.length; j++) {
-                    const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
-                }
-
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
                 resolve(files_to_download);
            }
         });
@@ -309,6 +274,43 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+async function handleOutputJSON(output, sub, user_uid) {
+    if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+        await setFreshUploads(sub, user_uid);
+        checkVideosForFreshUploads(sub, user_uid);
+    }
+
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+        logger.verbose('No additional videos to download for ' + sub.name);
+        return [];
+    }
+
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
+    const files_to_download = await getFilesToDownload(sub, output_jsons);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+    for (let j = 0; j < files_to_download.length; j++) {
+        const file_to_download = files_to_download[j];
+        file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
+    }
+
+    return files_to_download;
+}
+
 function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
@@ -319,10 +321,10 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
 
     const base_download_options = {
-        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        maxHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
         customFileFolderPath: getAppendedBasePath(sub, basePath),
         customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        customArchivePath: path.join(basePath, 'archives', sub.name),
         additionalArgs: sub.custom_args
     }
 
@@ -337,8 +339,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -364,6 +364,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
+    // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
+        const archive_path = path.join(appendedBasePath, 'archive.txt');
+        await fs.writeFile(archive_path, archive_text);
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
@@ -374,26 +384,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
-    let archive_dir = null;
-    let archive_path = null;
-
-    if (useArchive && !redownload) {
-        if (sub.archive) {
-            archive_dir = sub.archive;
-            if (sub.type && sub.type === 'audio') {
-                archive_path = path.join(archive_dir, 'merged_audio.txt');
-            } else {
-                archive_path = path.join(archive_dir, 'merged_video.txt');
-            }            
-        }
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
-    // if streaming only mode, just get the list of videos
-    if (sub.streamingOnly) {
-        downloadConfig = ['-f', 'best', '--dump-json'];
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -418,9 +408,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
     if (default_downloader === 'yt-dlp') {
-        downloadConfig.push('--no-clean-infojson');
+        downloadConfig.push('--no-clean-info-json');
     }
 
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
     return downloadConfig;
 }
 
@@ -434,7 +426,14 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+                continue;
             }
+            const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useYoutubeDLArchive) {
+                const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+                if (exists_in_archive) continue;
+            }
+
             files_to_download.push(output_json);
         }
     }
@@ -453,7 +452,12 @@ async function getAllSubscriptions() {
 }
 
 async function getSubscription(subID) {
-    return await db_api.getRecord('subscriptions', {id: subID});
+    // stringify and parse because we may override the 'downloading' property
+    const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
+    // now with the download_queue, we may need to override 'downloading'
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
+    return sub;
 }
 
 async function getSubscriptionByName(subName, user_uid = null) {
@@ -467,7 +471,7 @@ async function updateSubscription(sub) {
 
 async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
-        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
+        await updateSubscriptionProperty(sub, assignment_obj);
     });
 }
 
@@ -479,6 +483,7 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
 
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    if (!sub_files) return;
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
     sub_files.forEach(async file => {
         if (current_date === file['upload_date'].replace(/-/g, '')) {

backend/tasks.js

@@ -1,5 +1,7 @@
 const db_api = require('./db');
+const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
 
 const fs = require('fs-extra');
 const logger = require('./logger');
@@ -33,6 +35,28 @@ const TASKS = {
         confirm: youtubedl_api.updateYoutubeDL,
         title: 'Update youtube-dl',
         job: null
+    },
+    delete_old_files: {
+        run: checkForAutoDeleteFiles,
+        confirm: autoDeleteFiles,
+        title: 'Delete old files',
+        job: null
+    },
+    import_legacy_archives: {
+        run: archive_api.importArchives,
+        title: 'Import legacy archives',
+        job: null
+    }
+}
+
+const defaultOptions = {
+    all: {
+        auto_confirm: false
+    },
+    delete_old_files: {
+        blacklist_files: false,
+        blacklist_subscription_files: false,
+        threshold_days: ''
     }
 }
 
@@ -45,7 +69,7 @@ function scheduleJob(task_key, schedule) {
         const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
         const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
         const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
         return null;
@@ -57,7 +81,7 @@ function scheduleJob(task_key, schedule) {
             logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
             return;
         }
-        
+
         // remove schedule if it's a one-time task
         if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
         // we're just "running" the task, any confirmation should be user-initiated
@@ -77,9 +101,10 @@ exports.setupTasks = async () => {
     const tasks_keys = Object.keys(TASKS);
     for (let i = 0; i < tasks_keys.length; i++) {
         const task_key = tasks_keys[i];
+        const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
         const task_in_db = await db_api.getRecord('tasks', {key: task_key});
         if (!task_in_db) {
-            // insert task metadata into table if missing
+            // insert task metadata into table if missing, eventually move title to UI
             await db_api.insertRecordIntoTable('tasks', {
                 key: task_key,
                 title: TASKS[task_key]['title'],
@@ -90,9 +115,19 @@ exports.setupTasks = async () => {
                 data: null,
                 error: null,
                 schedule: null,
-                options: {}
+                options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
             });
         } else {
+            // verify all options exist in task
+            for (const key of Object.keys(mergedDefaultOptions)) {
+                const option_key = `options.${key}`;
+                // Remove any potential mangled option keys (#861)
+                await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
+                if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
+                    await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
+                }
+            }
+
             // reset task if necessary
             await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
 
@@ -123,15 +158,23 @@ exports.executeTask = async (task_key) => {
 
 exports.executeRun = async (task_key) => {
     logger.verbose(`Running task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     // don't set running to true when backup up DB as it will be stick "running" if restored
     if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
     const data = await TASKS[task_key].run();
     await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
     logger.verbose(`Finished running task ${task_key}`);
+    const task_obj = await db_api.getRecord('tasks', {key: task_key});
+    await notifications_api.sendTaskNotification(task_obj, false);
+
+    if (task_obj['options'] && task_obj['options']['auto_confirm']) {
+        exports.executeConfirm(task_key);
+    }
 }
 
 exports.executeConfirm = async (task_key) => {
     logger.verbose(`Confirming task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     if (!TASKS[task_key]['confirm']) {
         return null;
     }
@@ -141,6 +184,7 @@ exports.executeConfirm = async (task_key) => {
     await TASKS[task_key].confirm(data);
     await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
     logger.verbose(`Finished confirming task ${task_key}`);
+    await notifications_api.sendTaskNotification(task_obj, false);
 }
 
 exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -193,4 +237,31 @@ async function removeDuplicates(data) {
     }
 }
 
+// auto delete files
+
+async function checkForAutoDeleteFiles() {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
+        const error_message = 'Failed to do delete check because no limit was set!';
+        logger.error(error_message);
+        await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
+        return null;
+    }
+    const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
+    const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
+    const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
+    return {files_to_remove: files_to_remove};
+}
+
+async function autoDeleteFiles(data) {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (data['files_to_remove']) {
+        logger.info(`Removing ${data['files_to_remove'].length} old files!`);
+        for (let i = 0; i < data['files_to_remove'].length; i++) {
+            const file_to_remove = data['files_to_remove'][i];
+            await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
+        }
+    }
+}
+
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -1,8 +1,9 @@
-var assert = require('assert');
+/* eslint-disable no-undef */
+const assert = require('assert');
 const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
 
-process.chdir('./backend')
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -37,8 +38,11 @@ var auth_api = require('../authentication/auth');
 var db_api = require('../db');
 const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
+const archive_api = require('../archive');
+const categories_api = require('../categories');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
 
 db_api.initialize(db, users_db);
 
@@ -64,12 +68,12 @@ const sample_video_json = {
 
 describe('Database', async function() {
     describe('Import', async function() {
-        it('Migrate', async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords();
-            const success = await db_api.importJSONToDB(db.value(), users_db.value());
-            assert(success);
-        });
+        // it('Migrate', async function() {
+        //     await db_api.connectToDB();
+        //     await db_api.removeAllRecords();
+        //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
+        //     assert(success);
+        // });
 
         it('Transfer to remote', async function() {
             await db_api.removeAllRecords('test');
@@ -102,157 +106,208 @@ describe('Database', async function() {
         });
     });
 
-    describe('Export', function() {
-
-    });
-
-
     describe('Basic functions', async function() {
-        beforeEach(async function() {
-            await db_api.connectToDB();
-            await db_api.removeAllRecords('test');
-        });
-        it('Add and read record', async function() {
-            this.timeout(120000);
-            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
-            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
-            assert(added_record['test_add'] === 'test');
-            await db_api.removeRecord('test', {test_add: 'test'});
-        });
+        
+        // test both local_db and remote_db
+        const local_db_modes = [false, true];
 
-        it('Find duplicates by key', async function() {
-            const test_duplicates = [
-                {
-                    test: 'testing',
-                    key: '1'
-                },
-                {
-                    test: 'testing',
-                    key: '2'
-                },
-                {
-                    test: 'testing_missing',
-                    key: '3'
-                },
-                {
-                    test: 'testing',
-                    key: '4'
-                }
-            ];
-            await db_api.insertRecordsIntoTable('test', test_duplicates);
-            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-            console.log(duplicates);
-        });
-
-        it('Update record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
-            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
-            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
-            assert(updated_record['added_field']);
-            await db_api.removeRecord('test', {test_update: 'test'});
-        });
-
-        it('Remove record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
-            assert(delete_succeeded);
-            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
-            assert(!deleted_record);
-        });
-
-        it('Push to record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
-            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 1);
-        });
-
-        it('Pull from record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
-            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 0);
-        });
-
-        it('Bulk add', async function() {
-            this.timeout(120000);
-            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
-            const test_records = [];
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                test_records.push({
-                    uid: uuid()
+        for (const local_db_mode of local_db_modes) {
+            let use_local_db = local_db_mode;
+            describe(`Use local DB - ${use_local_db}`, async function() {
+                beforeEach(async function() {
+                    if (!use_local_db) {
+                        this.timeout(120000);
+                        await db_api.connectToDB(0);
+                    }
+                    await db_api.removeAllRecords('test');
                 });
-            }
-            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-
-            const received_records = await db_api.getRecords('test');
-            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
-        });
-
-        it('Bulk update', async function() {
-            // bulk add records
-            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
-            const test_records = [];
-            const update_obj = {};
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const test_uid =  uuid();
-                test_records.push({
-                    uid: test_uid
+                it('Add and read record', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+                    assert(added_record['test_add'] === 'test');
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Add and read record - Nested property', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
+                    const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
+                    assert(added_record['test_add'] === 'test');
+                    assert(!not_added_record);
+                    await db_api.removeRecord('test', {test_add: 'test'});
+                });
+                it('Replace filter', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
+                    assert(count === 1);
+                    await db_api.removeRecord('test', {test_replace_filter: 'test'});
+                });
+                it('Find duplicates by key', async function() {
+                    const test_duplicates = [
+                        {
+                            test: 'testing',
+                            key: '1'
+                        },
+                        {
+                            test: 'testing',
+                            key: '2'
+                        },
+                        {
+                            test: 'testing_missing',
+                            key: '3'
+                        },
+                        {
+                            test: 'testing',
+                            key: '4'
+                        }
+                    ];
+                    await db_api.insertRecordsIntoTable('test', test_duplicates);
+                    const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+                    console.log(duplicates);
                 });
-                update_obj[test_uid] = {added_field: true};
-            }
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            assert(success);
 
-            // makes sure they are added
-            const received_records = await db_api.getRecords('test');
-            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                it('Update record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+                    await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+                    const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+                    assert(updated_record['added_field']);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
-            assert(success);
+                it('Update records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
+                    await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
+                    const updated_records = await db_api.getRecords('test', {added_field: true});
+                    assert(updated_records.length === 2);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            const received_updated_records = await db_api.getRecords('test');
-            for (let i = 0; i < received_updated_records.length; i++) {
-                success &= received_updated_records[i]['added_field'];
-            }
-            assert(success);
-        });
+                it('Remove property from record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
+                    await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
+                    const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
+                    assert(updated_record['test_keep']);
+                    assert(!updated_record['test_remove']);
+                    await db_api.removeRecord('test', {test_keep: 'test'});
+                });
 
-        it('Stats', async function() {
-            const stats = await db_api.getDBStats();
-            assert(stats);
-        });
+                it('Remove record', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+                    assert(!deleted_record);
+                });
 
-        it('Query speed', async function() {
-            this.timeout(120000); 
-            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
-            const test_records = [];
-            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                const uid = uuid();
-                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
-            }
-            const insert_start = Date.now();
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            const insert_end = Date.now();
+                it('Remove records', async function() {
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+                    const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
+                    assert(delete_succeeded);
+                    const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
+                    assert(count === 0);
+                });
 
-            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+                it('Push to record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+                    await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 1);
+                });
 
-            const query_start = Date.now();
-            const random_record = await db_api.getRecord('test', {uid: random_uid});
-            const query_end = Date.now();
+                it('Pull from record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+                    await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 0);
+                });
 
-            console.log(random_record)
+                it('Bulk add', async function() {
+                    this.timeout(120000);
+                    const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+                    const test_records = [];
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        test_records.push({
+                            uid: uuid()
+                        });
+                    }
+                    const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
 
-            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+                    const received_records = await db_api.getRecords('test');
+                    assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                });
 
-            success = !!random_record;
+                it('Bulk update', async function() {
+                    // bulk add records
+                    const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+                    const test_records = [];
+                    const update_obj = {};
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const test_uid =  uuid();
+                        test_records.push({
+                            uid: test_uid
+                        });
+                        update_obj[test_uid] = {added_field: true};
+                    }
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    assert(success);
 
-            assert(success);
-        });
+                    // makes sure they are added
+                    const received_records = await db_api.getRecords('test');
+                    assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+                    success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
+                    assert(success);
+
+                    const received_updated_records = await db_api.getRecords('test');
+                    for (let i = 0; i < received_updated_records.length; i++) {
+                        success &= received_updated_records[i]['added_field'];
+                    }
+                    assert(success);
+                });
+
+                it('Stats', async function() {
+                    const stats = await db_api.getDBStats();
+                    assert(stats);
+                });
+
+                it('Query speed', async function() {
+                    this.timeout(120000); 
+                    const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+                    const test_records = [];
+                    let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const uid = uuid();
+                        if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                        test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                    }
+                    const insert_start = Date.now();
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    const insert_end = Date.now();
+
+                    console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+                    const query_start = Date.now();
+                    const random_record = await db_api.getRecord('test', {uid: random_uid});
+                    const query_end = Date.now();
+
+                    console.log(random_record)
+
+                    console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+                    success = !!random_record;
+
+                    assert(success);
+                });
+            });
+        }
     });
 
     describe('Local DB Filters', async function() {
@@ -287,8 +342,6 @@ describe('Multi User', async function() {
     const playlist_to_test = 'ysabVZz4x';
     beforeEach(async function() {
         await db_api.connectToDB();
-        auth_api.initialize(db_api, logger);
-        subscriptions_api.initialize(db_api, logger);
         user = await auth_api.login('admin', 'pass');
     });
     describe('Authentication', function() {
@@ -297,8 +350,10 @@ describe('Multi User', async function() {
         });
     });
     describe('Video player - normal', async function() {
-        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
-        await db_api.insertRecordIntoTable('files', sample_video_json);
+        beforeEach(async function() {
+            await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+            await db_api.insertRecordIntoTable('files', sample_video_json);
+        });
         const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
             const video_obj = await db_api.getVideo(video_to_test);
@@ -399,6 +454,19 @@ describe('Downloader', function() {
 
     });
 
+    it('Tag file', async function() {
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        const tags = {
+            title: sample_json['title'],
+            artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
+            TRCK: '27'
+        }
+        NodeID3.write(tags, audio_path);
+        const written_tags = NodeID3.read(audio_path);
+        assert(written_tags['raw']['TRCK'] === '27');
+    });
+
     it('Queue file', async function() {
         this.timeout(300000); 
         const returned_download = await downloader_api.createDownload(url, 'video', options);
@@ -442,14 +510,33 @@ describe('Downloader', function() {
         const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
         const updated_args1 = utils.injectArgs(original_args1, new_args1);
         const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
-        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
+        assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
 
         const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
         const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
         const updated_args2 = utils.injectArgs(original_args2, new_args2);
-        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
-        console.log(updated_args2);
-        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
+
+        const original_args3 = ['-o', '%(title)s.%(ext)s'];
+        const new_args3 =  ['--min-filesize','1'];
+        const updated_args3 = utils.injectArgs(original_args3, new_args3);
+        const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
+        assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
+    });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1710641401';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
     });
 });
 
@@ -521,7 +608,7 @@ describe('Tasks', function() {
         fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file, true);
+        assert(!!imported_file === true);
         
         // post-test cleanup
         if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -561,4 +648,207 @@ describe('Tasks', function() {
         const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
         assert(dummy_task_obj['data']);
     });
+});
+
+describe('Archive', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    it('Import archive', async function() {
+        const archive_text = `
+            testextractor1 testing1
+            testextractor1 testing2
+            testextractor2 testing1
+            testextractor1 testing3
+
+        `;
+        const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
+        assert(count === 4)
+        const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        console.log(archive_items);
+        assert(archive_items.length === 4);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
+
+        const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        assert(success);
+    });
+
+    it('Get archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+
+        assert(archive_item1 && archive_item2);
+    });
+
+    it('Archive duplicates', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
+
+        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
+        assert(count === 3);
+    });
+
+    it('Remove from archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
+
+        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
+        assert(success);
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        assert(!!archive_item1);
+        
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+        assert(!archive_item2);
+
+        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
+        assert(!!archive_item3);
+    });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
+    });
+
+    it('Convert flat object to nested object', async function() {
+        // No modfication
+        const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
+        const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
+        assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
+        assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
+
+        // Standard setup
+        const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
+        const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
+        assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
+        assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
+
+        // Nested branches
+        const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
+        const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
+    });
+});
+
+describe('Categories', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        const new_category = {
+            name: 'test_category',
+            uid: uuid(),
+            rules: [],
+            custom_output: ''
+        };
+    
+        await db_api.insertRecordIntoTable('categories', new_category);
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('categories', {name: 'test_category'});
+    });
+
+    it('Categorize - includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        console.log(category);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - AND', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'and',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - OR', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'or',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category);
+    });
 });

backend/twitch.js

@@ -1,90 +1,73 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
 const config_api = require('./config');
+const logger = require('./logger');
 
-async function getCommentsForVOD(clientID, vodId) {
-    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
-        batch,
-        cursor;
+const moment = require('moment');
+const fs = require('fs-extra')
+const path = require('path');
+const { promisify } = require('util');
+const child_process = require('child_process');
 
-    let comments = null;
-
-    try {
-        do {
-            batch = (await Axios.get(url, {
-                headers: {
-                    'Client-ID': clientID,
-                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
-                    'Content-Type': 'application/json; charset=UTF-8',
-                }
-            })).data;
-
-            const str = batch.comments.map(c => {
-                let {
-                        created_at: msgCreated,
-                        content_offset_seconds: timestamp,
-                        commenter: {
-                            name,
-                            _id,
-                            created_at: acctCreated
-                        },
-                        message: {
-                            body: msg,
-                            user_color: user_color
-                        }
-                    } = c;
-
-                const timestamp_str = moment.duration(timestamp, 'seconds')
-                    .toISOString()
-                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
-                        (_, ...ms) => {
-                            const seg = v => v ? v.padStart(2, '0') : '00';
-                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
-                        });
-
-                acctCreated = moment(acctCreated).utc();
-                msgCreated = moment(msgCreated).utc();
-
-                if (!comments) comments = [];
-
-                comments.push({
-                    timestamp: timestamp,
-                    timestamp_str: timestamp_str,
-                    name: name,
-                    message: msg,
-                    user_color: user_color
-                });
-                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
-                // return line;
-            }).join('\n');
-
-            cursor = batch._next;
-            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
-            await new Promise(res => setTimeout(res, 300));
-        } while (cursor);
-    } catch (err) {
-        console.error(err);
+async function getCommentsForVOD(vodId) {
+    const exec = promisify(child_process.exec);
+    
+    // Reject invalid params to prevent command injection attack
+    if (!vodId.match(/^[0-9a-z]+$/)) {
+        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
+        return null;
     }
 
-    return comments;
+    const is_windows = process.platform === 'win32';
+    const cliExt = is_windows ? '.exe' : ''
+    const cliPath = `TwitchDownloaderCLI${cliExt}`
+
+    if (!fs.existsSync(cliPath)) {
+        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
+        return null;
+    }
+
+    const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
+
+    if (result['stderr']) {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
+        logger.error(result['stderr']);
+        return null;
+    }
+
+    const temp_chat_path = path.join('appdata', `${vodId}.json`);
+
+    const raw_json = fs.readJSONSync(temp_chat_path);
+    const new_json = raw_json.comments.map(comment_obj => {
+        return {
+            timestamp: comment_obj.content_offset_seconds,
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+            name: comment_obj.commenter.name,
+            message: comment_obj.message.body,
+            user_color: comment_obj.message.user_color
+        }
+    });
+
+    fs.unlinkSync(temp_chat_path);
+
+    return new_json;
 }
 
 async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
     let file_path = null;
 
     if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
         }
     }
 
@@ -96,23 +79,26 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
     return chat_file;
 }
 
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
-    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
-    const chat = await getCommentsForVOD(twitch_api_key, vodId);
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const chat = await getCommentsForVOD(vodId);
 
     // save file if needed params are included
     let file_path = null;
-    if (user_uid) {
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            file_path = path.join(type, `${id}.twitch_chat.json`);
         }
     }
 
@@ -121,6 +107,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
     return chat;
 }
 
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
 module.exports = {
     getCommentsForVOD: getCommentsForVOD,
     getTwitchChatByFileID: getTwitchChatByFileID,

backend/utils.js

@@ -4,6 +4,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const archiver = require('archiver');
 const fetch = require('node-fetch');
 const ProgressBar = require('progress');
+const winston = require('winston');
 
 const config_api = require('./config');
 const logger = require('./logger');
@@ -12,7 +13,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-function getTrueFileName(unfixed_path, type) {
+exports.getTrueFileName = (unfixed_path, type) => {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -27,13 +28,13 @@ function getTrueFileName(unfixed_path, type) {
     return fixed_path;
 }
 
-async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
+exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
     // return empty array if the path doesn't exist
     if (!(await fs.pathExists(basePath))) return [];
 
     let files = [];
     const ext = type === 'audio' ? 'mp3' : 'mp4';
-    var located_files = await recFindByExt(basePath, ext);
+    var located_files = await exports.recFindByExt(basePath, ext);
     for (let i = 0; i < located_files.length; i++) {
         let file = located_files[i];
         var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -41,33 +42,33 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
         var stats = await fs.stat(file);
 
         var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await getJSONByType(type, id, basePath);
+        var jsonobj = await exports.getJSONByType(type, id, basePath);
         if (!jsonobj) continue;
         if (full_metadata) {
             jsonobj['id'] = id;
             files.push(jsonobj);
             continue;
         }
-        var upload_date = formatDateString(jsonobj.upload_date);
+        var upload_date = exports.formatDateString(jsonobj.upload_date);
 
         var isaudio = type === 'audio';
-        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+        var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
                                 stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
 }
 
-async function createContainerZipFile(file_name, container_file_objs) {
+exports.createContainerZipFile = async (file_name, container_file_objs) => {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
 }
 
-async function createZipFile(zip_file_path, file_paths) {
+exports.createZipFile = async (zip_file_path, file_paths) => {
     let output = fs.createWriteStream(zip_file_path);
 
     var archive = archiver('zip', {
@@ -91,11 +92,11 @@ async function createZipFile(zip_file_path, file_paths) {
     await archive.finalize();
 
     // wait a tiny bit for the zip to reload in fs
-    await wait(100);
+    await exports.wait(100);
     return zip_file_path;
 }
 
-function getJSONMp4(name, customPath, openReadPerms = false) {
+exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -110,7 +111,7 @@ function getJSONMp4(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSONMp3(name, customPath, openReadPerms = false) {
+exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
     var obj = null;
     if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -127,11 +128,11 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSON(file_path, type) {
+exports.getJSON = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
     let obj = null;
-    var jsonPath = removeFileExtension(file_path) + '.info.json';
-    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
+    var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
     if (fs.existsSync(jsonPath))
     {
         obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -142,12 +143,12 @@ function getJSON(file_path, type) {
     return obj;
 }
 
-function getJSONByType(type, name, customPath, openReadPerms = false) {
-    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
+    return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(file_path) {
-    const file_path_no_extension = removeFileExtension(file_path);
+exports.getDownloadedThumbnail = (file_path) => {
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
     let webpPath = file_path_no_extension + '.webp';
@@ -163,7 +164,7 @@ function getDownloadedThumbnail(file_path) {
         return null;
 }
 
-function getExpectedFileSize(input_info_jsons) {
+exports.getExpectedFileSize = (input_info_jsons) => {
     // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
     const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
 
@@ -172,11 +173,13 @@ function getExpectedFileSize(input_info_jsons) {
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
-            info_json.formats.forEach(available_format => {
-                if (available_format.format_id === format_id && available_format.filesize) {
-                    individual_expected_filesize += available_format.filesize;
-                }
-            });
+            if (info_json.formats !== undefined) {
+                info_json.formats.forEach(available_format => {
+                  if (available_format.format_id === format_id && (available_format.filesize || available_format.filesize_approx)) {
+                    individual_expected_filesize += (available_format.filesize ? available_format.filesize : available_format.filesize_approx);
+                  }
+                });
+            }
         });
         expected_filesize += individual_expected_filesize;
     });
@@ -184,12 +187,12 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(file_path, type) {
+exports.fixVideoMetadataPerms = (file_path, type) => {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     const files_to_fix = [
         // JSONs
@@ -206,11 +209,11 @@ function fixVideoMetadataPerms(file_path, type) {
     }
 }
 
-function deleteJSONFile(file_path, type) {
+exports.deleteJSONFile = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
-    
+    const file_path_no_extension = exports.removeFileExtension(file_path);
+
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
 
@@ -218,33 +221,7 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-async function removeIDFromArchive(archive_path, id) {
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
-    if (!data) {
-        logger.error('Archive could not be found.');
-        return;
-    }
-
-    let dataArray = data.split('\n'); // convert file data in an array
-    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-    let lastIndex = -1; // let say, we have not found the keyword
-
-    for (let index=0; index<dataArray.length; index++) {
-        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-            lastIndex = index; // found a line includes a id keyword
-            break;
-        }
-    }
-
-    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-
-    // UPDATE FILE WITH NEW DATA
-    const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
-    if (line) return line;
-}
-
-function durationStringToNumber(dur_str) {
+exports.durationStringToNumber = (dur_str) => {
     if (typeof dur_str === 'number') return dur_str;
     let num_sum = 0;
     const dur_str_parts = dur_str.split(':');
@@ -254,23 +231,22 @@ function durationStringToNumber(dur_str) {
     return num_sum;
 }
 
-function getMatchingCategoryFiles(category, files) {
+exports.getMatchingCategoryFiles = (category, files) => {
     return files && files.filter(file => file.category && file.category.uid === category.uid);
 }
 
-function addUIDsToCategory(category, files) {
-    const files_that_match = getMatchingCategoryFiles(category, files);
+exports.addUIDsToCategory = (category, files) => {
+    const files_that_match = exports.getMatchingCategoryFiles(category, files);
     category['uids'] = files_that_match.map(file => file.uid);
     return files_that_match;
 }
 
-function getCurrentDownloader() {
+exports.getCurrentDownloader = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     return details_json['downloader'];
 }
 
-async function recFindByExt(base, ext, files, result, recursive = true)
-{
+exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
     files = files || (await fs.readdir(base))
     result = result || []
 
@@ -279,7 +255,7 @@ async function recFindByExt(base, ext, files, result, recursive = true)
         if ( (await fs.stat(newbase)).isDirectory() )
         {
             if (!recursive) continue;
-            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+            result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
         {
@@ -292,23 +268,23 @@ async function recFindByExt(base, ext, files, result, recursive = true)
     return result
 }
 
-function removeFileExtension(filename) {
+exports.removeFileExtension = (filename) => {
     const filename_parts = filename.split('.');
     filename_parts.splice(filename_parts.length - 1);
     return filename_parts.join('.');
 }
 
-function formatDateString(date_string) {
+exports.formatDateString = (date_string) => {
     return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
 }
 
-function createEdgeNGrams(str) {
+exports.createEdgeNGrams = (str) => {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
-        
+
         return str.split(" ").reduce((ngrams, token) => {
-            if (token.length > minGram) {   
+            if (token.length > minGram) {
                 for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
                     ngrams = [...ngrams, token.substr(0, i)]
                 }
@@ -318,13 +294,13 @@ function createEdgeNGrams(str) {
             return ngrams
         }, []).join(" ")
     }
-    
+
     return str
 }
 
 // ffmpeg helper functions
 
-async function cropFile(file_path, start, end, ext) {
+exports.cropFile = async (file_path, start, end, ext) => {
     return new Promise(resolve => {
         const temp_file_path = `${file_path}.cropped${ext}`;
         let base_ffmpeg_call = ffmpeg(file_path);
@@ -353,13 +329,13 @@ async function cropFile(file_path, start, end, ext) {
  * setTimeout, but its a promise.
  * @param {number} ms
  */
- async function wait(ms) {
+exports.wait = async (ms) => {
     await new Promise(resolve => {
         setTimeout(resolve, ms);
     });
 }
 
-async function checkExistsWithTimeout(filePath, timeout) {
+exports.checkExistsWithTimeout = async (filePath, timeout) => {
     return new Promise(function (resolve, reject) {
 
         var timer = setTimeout(function () {
@@ -388,7 +364,7 @@ async function checkExistsWithTimeout(filePath, timeout) {
 }
 
 // helper function to download file using fetch
-async function fetchFile(url, path, file_label) {
+exports.fetchFile = async (url, path, file_label) => {
     var len = null;
     const res = await fetch(url);
 
@@ -415,10 +391,10 @@ async function fetchFile(url, path, file_label) {
     });
 }
 
-async function restartServer(is_update = false) {
+exports.restartServer = async (is_update = false) => {
     logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
 
-    // the following line restarts the server through nodemon
+    // the following line restarts the server through pm2
     fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
     process.exit(1);
 }
@@ -428,20 +404,21 @@ async function restartServer(is_update = false) {
 //  - if already exists and doesn't have value, ignore
 //  - if it doesn't exist and has value, add both arg and value
 //  - if it doesn't exist and doesn't have value, add arg
-function injectArgs(original_args, new_args) {
+exports.injectArgs = (original_args, new_args) => {
     const updated_args = original_args.slice();
     try {
         for (let i = 0; i < new_args.length; i++) {
             const new_arg = new_args[i];
             if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
-            
+
             if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
                 if (original_args.includes(new_arg)) {
                     const original_index = original_args.indexOf(new_arg);
-                    original_args.splice(original_index, 2);
+                    updated_args.splice(original_index, 2);
                 }
 
                 updated_args.push(new_arg, new_args[i + 1]);
+                i++; // we need to skip the arg value on the next loop
             } else {
                 if (!original_args.includes(new_arg)) {
                     updated_args.push(new_arg);
@@ -456,7 +433,11 @@ function injectArgs(original_args, new_args) {
     return updated_args;
 }
 
-const searchObjectByString = function(o, s) {
+exports.filterArgs = (args, args_to_remove) => {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+exports.searchObjectByString = (o, s) => {
     s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
     s = s.replace(/^\./, '');           // strip a leading dot
     var a = s.split('.');
@@ -471,6 +452,73 @@ const searchObjectByString = function(o, s) {
     return o;
 }
 
+exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
+exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
+exports.getBaseURL = () => {
+    return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
+}
+
+exports.updateLoggerLevel = (new_logger_level) => {
+    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    if (!possible_levels.includes(new_logger_level)) {
+        logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
+        new_logger_level = 'info';
+    }
+    logger.level = new_logger_level;
+    winston.loggers.get('console').level = new_logger_level;
+    logger.transports[2].level = new_logger_level;
+}
+
+exports.convertFlatObjectToNestedObject = (obj) => {
+    const result = {};
+    for (const key in obj) {
+      const nestedKeys = key.split('.');
+      let currentObj = result;
+      for (let i = 0; i < nestedKeys.length; i++) {
+        if (i === nestedKeys.length - 1) {
+          currentObj[nestedKeys[i]] = obj[key];
+        } else {
+          currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
+          currentObj = currentObj[nestedKeys[i]];
+        }
+      }
+    }
+    return result;
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -488,33 +536,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.view_count = view_count;
     this.height = height;
     this.abr = abr;
-}
+    this.favorite = false;
+}   
+exports.File = File;
 
-module.exports = {
-    getJSONMp3: getJSONMp3,
-    getJSONMp4: getJSONMp4,
-    getJSON: getJSON,
-    getTrueFileName: getTrueFileName,
-    getDownloadedThumbnail: getDownloadedThumbnail,
-    getExpectedFileSize: getExpectedFileSize,
-    fixVideoMetadataPerms: fixVideoMetadataPerms,
-    deleteJSONFile: deleteJSONFile,
-    removeIDFromArchive: removeIDFromArchive,
-    getDownloadedFilesByType: getDownloadedFilesByType,
-    createContainerZipFile: createContainerZipFile,
-    durationStringToNumber: durationStringToNumber,
-    getMatchingCategoryFiles: getMatchingCategoryFiles,
-    getCurrentDownloader: getCurrentDownloader,
-    recFindByExt: recFindByExt,
-    removeFileExtension: removeFileExtension,
-    formatDateString: formatDateString,
-    cropFile: cropFile,
-    createEdgeNGrams: createEdgeNGrams,
-    wait: wait,
-    checkExistsWithTimeout: checkExistsWithTimeout,
-    fetchFile: fetchFile,
-    restartServer: restartServer,
-    injectArgs: injectArgs,
-    searchObjectByString: searchObjectByString,
-    File: File
-}

backend/youtube-dl.js

@@ -90,7 +90,7 @@ exports.updateYoutubeDL = async (latest_update_version) => {
 
 exports.verifyBinaryExistsLinux = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
-    if (!is_windows && details_json && (details_json['path'].includes('.exe') || !details_json['path'])) {
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
         details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
         details_json['exec'] = 'youtube-dl';
         details_json['version'] = OUTDATED_VERSION;

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.2"
+appVersion: "4.3.1"

chart/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018 Isaac Grynsztein
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

chart/README.md

@@ -0,0 +1,3 @@
+The official YoutubeDL-Material chart
+
+Repo: https://github.com/Tzahi12345/YoutubeDL-Material

chart/artifacthub-repo.yml

@@ -0,0 +1,4 @@
+# Artifact Hub repository metadata file
+repositoryID: ff79ae03-57c1-4f18-8368-5e085d06e2f1
+owners:
+  - name: tzahi12345

chart/index.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+entries:
+  youtubedl-material:
+  - apiVersion: v1
+    created: "2021-01-01T05:51:36.304331-05:00"
+    description: A Material Design frontend for youtube-dl
+    digest: f7340d24fb051ade30b890db3b5c483a884b8459316dcf2ffc6fb9413d41252e
+    home: https://github.com/Tzahi12345/YoutubeDL-Material/
+    icon: https://i.imgur.com/IKOlr0N.png
+    keywords:
+    - youtubedl-material
+    - youtube-dl
+    maintainers:
+    - email: IsaacMGrynsztein@gmail.com
+      name: tzahi12345
+    name: youtubedl-material
+    sources:
+    - https://github.com/Tzahi12345/YoutubeDL-Material/
+    urls:
+    - youtubedl-material-0.0.1.tgz
+    version: 0.0.1
+generated: "2021-01-01T05:51:36.3003569-05:00"

chart/templates/claim-appdata-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-appdata
+  name: ytdl-material-claim-appdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "1Gi" .Values.appdataClaimSize }}
+status: {}

chart/templates/claim-audio-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-audio
+  name: ytdl-material-claim-audio
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.audioClaimSize }}
+status: {}

chart/templates/claim-subscriptions-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-subscriptions
+  name: ytdl-material-claim-subscriptions
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.subscriptionsClaimSize }}
+status: {}

chart/templates/claim-users-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-users
+  name: ytdl-material-claim-users
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.usersClaimSize }}
+status: {}

chart/templates/claim-video-persistentvolumeclaim.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: ytdl-material-claim-video
+  name: ytdl-material-claim-video
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ default "50Gi" .Values.videoClaimSize }}
+status: {}

docker-compose.yml

@@ -2,7 +2,6 @@ version: "2"
 services:
     ytdl_material:
         environment: 
-            ALLOW_CONFIG_MUTATIONS: 'true'
             ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
             ytdl_use_local_db: 'false'
             write_ytdl_config: 'true'
@@ -17,14 +16,12 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:nightly
+        image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        image: mongo
-        ports:
-            - "27017:27017"
+        image: mongo:4
         logging:
             driver: "none"          
         container_name: mongo-db
         restart: always
         volumes:
-            - ./db/:/data/db
+            - ./db/:/data/db

docker-utils/GetTwitchDownloader.py

@@ -0,0 +1,53 @@
+import platform
+import requests
+import shutil
+import os
+import re
+
+from github import Github
+
+machine = platform.machine()
+
+def isARM():
+    return True if machine.startswith('arm') else False
+
+def getLatestFileInRepo(repo, search_string):
+    # Create an unauthenticated instance of the Github object
+    g = Github(os.environ.get('GH_TOKEN'))
+
+    # Replace with the repository owner and name
+    repo = g.get_repo(repo)
+
+    # Get all releases of the repository
+    releases = repo.get_releases()
+
+    # Loop through the releases in reverse order (from latest to oldest)
+    for release in list(releases):
+        # Get the release assets (files attached to the release)
+        assets = release.get_assets()
+
+        # Loop through the assets
+        for asset in assets:
+            if re.search(search_string, asset.name):
+                print(f'Downloading: {asset.name}')
+                response = requests.get(asset.browser_download_url)
+                with open(asset.name, 'wb') as f:
+                    f.write(response.content)
+                print(f'Download complete: {asset.name}. Unzipping...')
+                shutil.unpack_archive(asset.name, './')
+                print(f'Unzipping complete!')
+                os.remove(asset.name)
+                break
+        else:
+            continue
+        break
+    else:
+        # If no matching release is found, print a message
+        print(f'No release found with {search_string}')
+
+def getLatestCLIRelease():
+    isArm = isARM()
+    searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
+    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
+
+getLatestCLIRelease()

docker-utils/ffmpeg-fetch.sh

@@ -26,7 +26,7 @@ apt-get update && apt-get -y install curl xz-utils
 echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
 curl -o ffmpeg.txz \
     --connect-timeout 5 \
-    --max-time 10 \
+    --max-time 120 \
     --retry 5 \
     --retry-delay 0 \
     --retry-max-time 40 \

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.2.0",
+  "version": "4.3.1",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -13,7 +13,7 @@
     "e2e": "ng e2e",
     "electron": "ng build --base-href ./ && electron .",
     "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
-    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n"
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n --out-file=messages.en.xlf"
   },
   "engines": {
     "node": "12.3.1",
@@ -21,62 +21,62 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
-    "@angular/animations": "^13.3.4",
-    "@angular/cdk": "^13.3.4",
-    "@angular/common": "^13.3.4",
-    "@angular/compiler": "^13.3.4",
-    "@angular/core": "^13.3.4",
-    "@angular/forms": "^13.3.4",
-    "@angular/localize": "^13.3.4",
-    "@angular/material": "^13.3.4",
-    "@angular/platform-browser": "^13.3.4",
-    "@angular/platform-browser-dynamic": "^13.3.4",
-    "@angular/router": "^13.3.4",
+    "@angular-devkit/core": "^15.0.1",
+    "@angular/animations": "^15.0.1",
+    "@angular/cdk": "^15.0.0",
+    "@angular/common": "^15.0.1",
+    "@angular/compiler": "^15.0.1",
+    "@angular/core": "^15.0.1",
+    "@angular/forms": "^15.0.1",
+    "@angular/localize": "^15.0.1",
+    "@angular/material": "^15.0.0",
+    "@angular/platform-browser": "^15.0.1",
+    "@angular/platform-browser-dynamic": "^15.0.1",
+    "@angular/router": "^15.0.1",
     "@fontsource/material-icons": "^4.5.4",
     "@ngneat/content-loader": "^5.0.0",
-    "@videogular/ngx-videogular": "^5.0.1",
+    "@videogular/ngx-videogular": "^6.0.0",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
-    "filesize": "^6.1.0",
+    "filesize": "^10.0.7",
     "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
-    "ng-lazyload-image": "^7.0.1",
-    "ngx-avatars": "^1.3.1",
+    "ngx-avatars": "^1.4.1",
     "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
-    "rxjs-compat": "^6.0.0-rc.0",
+    "rxjs-compat": "^6.6.7",
     "tslib": "^2.0.0",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "xliff-to-json": "^1.0.4",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
-    "@angular/cli": "^13.3.3",
-    "@angular/compiler-cli": "^13.3.4",
-    "@angular/language-service": "^13.3.4",
+    "@angular-devkit/build-angular": "^15.0.1",
+    "@angular/cli": "^15.0.1",
+    "@angular/compiler-cli": "^15.0.1",
+    "@angular/language-service": "^15.0.1",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "~3.6.0",
+    "@types/jasmine": "^4.3.1",
     "@types/node": "^12.11.1",
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
+    "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
-    "electron": "^13.6.6",
+    "electron": "^19.1.9",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.3.16",
+    "karma": "~6.4.2",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
-    "karma-jasmine": "~4.0.0",
+    "karma-jasmine": "~5.1.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.21.0",
+    "openapi-typescript-codegen": "^0.23.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
 export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
@@ -26,8 +27,10 @@ export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
 export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
 export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
 export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -40,14 +43,18 @@ export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchCh
 export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
 export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
 export { FileType } from './models/FileType';
+export { FileTypeFilter } from './models/FileTypeFilter';
 export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
 export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
 export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
 export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
 export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetArchivesRequest } from './models/GetArchivesRequest';
+export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -61,6 +68,7 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
 export type { GetLogsResponse } from './models/GetLogsResponse';
 export type { GetMp3sResponse } from './models/GetMp3sResponse';
 export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
 export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
 export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
 export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -71,17 +79,24 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
 export type { LoginResponse } from './models/LoginResponse';
+export type { Notification } from './models/Notification';
+export { NotificationAction } from './models/NotificationAction';
+export { NotificationType } from './models/NotificationType';
 export type { Playlist } from './models/Playlist';
 export type { RegisterRequest } from './models/RegisterRequest';
 export type { RegisterResponse } from './models/RegisterResponse';
+export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
 export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
 export type { SharingToggle } from './models/SharingToggle';
+export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
 export type { SubscribeResponse } from './models/SubscribeResponse';
 export type { Subscription } from './models/Subscription';
@@ -105,8 +120,10 @@ export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';
 export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
+export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/Archive.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type Archive = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    title: string;
+    user_uid?: string;
+    sub_id?: string;
+    timestamp: number;
+    uid: string;
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -2,11 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
-
 export type CreatePlaylistRequest = {
     playlistName: string;
     uids: Array<string>;
-    type: FileType;
     thumbnailURL: string;
 };

src/api-types/models/DBInfoResponse.ts

@@ -14,5 +14,6 @@ subscriptions?: TableInfo;
 users?: TableInfo;
 roles?: TableInfo;
 download_queue?: TableInfo;
+archives?: TableInfo;
 };
 };

src/api-types/models/DatabaseFile.ts

@@ -26,8 +26,20 @@ export type DatabaseFile = {
     path: string;
     upload_date: string;
     uid: string;
+    user_uid?: string;
     sharingEnabled?: boolean;
     category?: Category;
     view_count?: number;
     local_view_count?: number;
+    sub_id?: string;
+    registered?: number;
+    /**
+     * In pixels, only for videos
+     */
+    height?: number;
+    /**
+     * In Kbps
+     */
+    abr?: number;
+    favorite: boolean;
 };

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type DeleteArchiveItemsRequest = {
+    archives: Array<Archive>;
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteNotificationRequest = {
+    uid: string;
+};

src/api-types/models/DeletePlaylistRequest.ts

@@ -2,9 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
-
 export type DeletePlaylistRequest = {
     playlist_id: string;
-    type: FileType;
 };

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,12 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { SubscriptionRequestData } from './SubscriptionRequestData';
-
 export type DeleteSubscriptionFileRequest = {
-    file: string;
-    file_uid?: string;
-    sub: SubscriptionRequestData;
+    file_uid: string;
     /**
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */

src/api-types/models/Download.ts

@@ -19,7 +19,12 @@ export type Download = {
      * Error text, set if download fails.
      */
     error?: string | null;
+    /**
+     * Error type, may or may not be set in case of an error
+     */
+    error_type?: string | null;
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
+    prefetched_info?: any;
 };

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,8 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { FileType } from './FileType';
+
 export type DownloadArchiveRequest = {
-    sub: {
-archive_dir: string;
-};
+    type?: FileType;
+    sub_id?: string;
 };

src/api-types/models/DownloadRequest.ts

@@ -35,10 +35,18 @@ export type DownloadRequest = {
      * Height of the video, if known
      */
     selectedHeight?: string;
+    /**
+     * Max height that should be used, useful for playlists. selectedHeight will override this.
+     */
+    maxHeight?: string;
     /**
      * Specify ffmpeg/avconv audio quality
      */
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
+    /**
+     * If using youtube-dl archive, download will ignore it
+     */
+    ignoreArchive?: boolean;
 };

src/api-types/models/FileTypeFilter.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum FileTypeFilter {
+    AUDIO_ONLY = 'audio_only',
+    VIDEO_ONLY = 'video_only',
+    BOTH = 'both',
+}

src/api-types/models/GetAllFilesRequest.ts

@@ -0,0 +1,24 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileTypeFilter } from './FileTypeFilter';
+import type { Sort } from './Sort';
+
+export type GetAllFilesRequest = {
+    sort?: Sort;
+    range?: Array<number>;
+    /**
+     * Filter files by title
+     */
+    text_search?: string;
+    file_type_filter?: FileTypeFilter;
+    /**
+     * If set to true, only gets favorites
+     */
+    favorite_filter?: boolean;
+    /**
+     * Include if you want to filter by subscription
+     */
+    sub_id?: string;
+};

src/api-types/models/GetArchivesRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type GetArchivesRequest = {
+    type?: FileType;
+    sub_id?: string;
+};

src/api-types/models/GetArchivesResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type GetArchivesResponse = {
+    archives: Array<Archive>;
+};

src/api-types/models/GetNotificationsResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Notification } from './Notification';
+
+export type GetNotificationsResponse = {
+    notifications?: Array<Notification>;
+};

src/api-types/models/GetPlaylistResponse.ts

@@ -2,11 +2,14 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
+import type { DatabaseFile } from './DatabaseFile';
 import type { Playlist } from './Playlist';
 
 export type GetPlaylistResponse = {
     playlist: Playlist;
-    type: FileType;
     success: boolean;
+    /**
+     * File objects for every uid in the playlist's uids property, in the same order
+     */
+    file_objs?: Array<DatabaseFile>;
 };

src/api-types/models/ImportArchiveRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type ImportArchiveRequest = {
+    archive: string;
+    type: FileType;
+    sub_id?: string;
+};

src/api-types/models/Notification.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { NotificationAction } from './NotificationAction';
+import type { NotificationType } from './NotificationType';
+
+export type Notification = {
+    type: NotificationType;
+    uid: string;
+    user_uid?: string;
+    action?: Array<NotificationAction>;
+    read: boolean;
+    data?: any;
+    timestamp: number;
+};

src/api-types/models/NotificationAction.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationAction {
+    PLAY = 'play',
+    RETRY_DOWNLOAD = 'retry_download',
+    VIEW_DOWNLOAD_ERROR = 'view_download_error',
+    VIEW_TASKS = 'view_tasks',
+}

src/api-types/models/NotificationType.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum NotificationType {
+    DOWNLOAD_COMPLETE = 'download_complete',
+    DOWNLOAD_ERROR = 'download_error',
+    TASK_FINISHED = 'task_finished',
+}

src/api-types/models/Playlist.ts

@@ -14,4 +14,5 @@ export type Playlist = {
     duration: number;
     user_uid?: string;
     auto?: boolean;
+    sharingEnabled?: boolean;
 };

src/api-types/models/RestartDownloadResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { SuccessObject } from './SuccessObject';
+
+export type RestartDownloadResponse = (SuccessObject & {
+new_download_uid?: string;
+});

src/api-types/models/Schedule.ts

@@ -9,6 +9,7 @@ dayOfWeek?: Array<number>;
 hour?: number;
 minute?: number;
 timestamp?: number;
+tz?: string;
 };
 };
 

src/api-types/models/SetNotificationsToReadRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type SetNotificationsToReadRequest = {
+    uids: Array<string>;
+};

src/api-types/models/Sort.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type Sort = {
+    /**
+     * Property to sort by
+     */
+    by?: string;
+    /**
+     * 1 for ascending, -1 for descending
+     */
+    order?: number;
+};

src/api-types/models/Subscription.ts

@@ -10,7 +10,6 @@ export type Subscription = {
     id: string;
     type: FileType;
     user_uid: string | null;
-    streamingOnly: boolean;
     isPlaylist: boolean;
     archive?: string;
     timerange?: string;

src/api-types/models/Task.ts

@@ -4,6 +4,7 @@
 
 export type Task = {
     key: string;
+    title?: string;
     last_ran: number;
     last_confirmed: number;
     running: boolean;
@@ -11,4 +12,5 @@ export type Task = {
     data: any;
     error: string;
     schedule: any;
+    options?: any;
 };

src/api-types/models/UpdateTaskOptionsRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UpdateTaskOptionsRequest = {
+    task_key: string;
+    new_options: any;
+};

src/api-types/models/UploadCookiesRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type UploadCookiesRequest = {
+    cookies: Blob;
+};

src/app/app-routing.module.ts

@@ -23,7 +23,7 @@ const routes: Routes = [
 ];
 
 @NgModule({
-  imports: [RouterModule.forRoot(routes, { useHash: true, relativeLinkResolution: 'legacy' })],
+  imports: [RouterModule.forRoot(routes, { useHash: true })],
   exports: [RouterModule]
 })
 export class AppRoutingModule { }

src/app/app.component.css

@@ -13,7 +13,6 @@
 }
 
 .theme-slide-toggle {
-    top: 2px;
     left: 10px;
     position: relative;
     pointer-events: none;
@@ -25,4 +24,20 @@
 
 .top-toolbar {
     height: 64px;
-}
+    background: unset;
+}
+
+::ng-deep .top-menu-button > span {
+    width: 85px;
+    height: 26px;
+}
+
+::ng-deep .mdc-switch {
+    outline: none !important;
+}
+
+::ng-deep .notifications-menu {
+    width: 30vw !important;
+    max-width: 100% !important;
+    min-width: 280px !important;
+}

src/app/app.component.html

@@ -1,37 +1,41 @@
 <div [style.background]="postsService.theme ? postsService.theme.background_color : null" style="width: 100%; height: 100%;">
     <div class="mat-elevation-z3" style="position: relative; z-index: 10;">
-        <mat-toolbar color="primary" class="sticky-toolbar top-toolbar">
-            <div class="flex-row" width="100%" height="100%">
-                <div class="flex-column" style="text-align: left; margin-top: 1px;">
-                    <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
-                    <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
-                </div>
-                <div class="flex-column" style="text-align: center; margin-top: 5px;">
-                    <div style="font-size: 22px; text-shadow: #141414 0.25px 0.25px 1px;">
-                        {{topBarTitle}}
+        <mat-toolbar class="sticky-toolbar top-toolbar">
+            <div class="container-fluid" style="padding-left: 0px; padding-right: 0px">
+                <div class="row" width="100%" height="100%">
+                    <div class="col-6" style="text-align: left; margin-top: 1px;">
+                        <div style="display: flex; align-items: center;">
+                            <button #hamburgerMenu style="outline: none" *ngIf="router.url.split(';')[0] !== '/player'" mat-icon-button aria-label="Toggle side navigation" (click)="toggleSidenav()"><mat-icon>menu</mat-icon></button>
+                            <button (click)="goBack()" *ngIf="router.url.split(';')[0] === '/player'" mat-icon-button><mat-icon>arrow_back</mat-icon></button>
+                            <div style="margin-left: 8px; display: inline-block;"><button mat-icon-button routerLink='/home'><img style="width: 32px;" src="assets/images/logo_128px.png"></button></div>
+                        </div>
+                    </div>
+                    <div class="col-6" style="text-align: right; align-items: flex-end; display: inline-block">
+                        <button *ngIf="postsService.config?.Extra.enable_notifications" [matMenuTriggerFor]="notificationsMenu" (menuOpened)="notificationMenuOpened()" mat-icon-button><mat-icon [matBadge]="notification_count" matBadgeColor="warn" matBadgeSize="small" *ngIf="notification_count > 0">notifications</mat-icon><mat-icon *ngIf="notification_count === 0">notifications_none</mat-icon></button>
+                        <mat-menu [classList]="'notifications-menu'" (close)="notificationMenuClosed()" #notificationsMenu="matMenu">
+                            <app-notifications #notifications (notificationCount)="notificationCountUpdate($event)" (click)="$event.stopPropagation()"></app-notifications>
+                        </mat-menu>
+                        <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
+                        <mat-menu #menuSettings="matMenu">
+                            <button class="top-menu-button" (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
+                                <mat-icon>person</mat-icon>
+                                <span i18n="Profile menu label">Profile</span>
+                            </button>
+                            <button *ngIf="postsService.config && postsService.config.Downloader.use_youtubedl_archive" class="top-menu-button" (click)="openArchivesDialog()" mat-menu-item>
+                                <mat-icon>topic</mat-icon>
+                                <span i18n="Archives menu label">Archives</span>
+                            </button>
+                            <button class="top-menu-button" (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
+                            <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
+                            <span i18n="Dark mode toggle label">Dark</span>
+                            <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
+                            </button>
+                            <button class="top-menu-button" (click)="openAboutDialog()" mat-menu-item>
+                                <mat-icon>info</mat-icon>
+                                <span i18n="About menu label">About</span>
+                            </button>
+                        </mat-menu>
                     </div>
-                </div>
-                <div class="flex-column" style="text-align: right; align-items: flex-end;">
-                    <button [matMenuTriggerFor]="menuSettings" mat-icon-button><mat-icon>more_vert</mat-icon></button>
-                    <mat-menu #menuSettings="matMenu">
-                        <button (click)="openProfileDialog()" *ngIf="postsService.isLoggedIn" mat-menu-item>
-                            <mat-icon>person</mat-icon>
-                            <span i18n="Profile menu label">Profile</span>
-                        </button>
-                        <button (click)="themeMenuItemClicked($event)" *ngIf="allowThemeChange" mat-menu-item>
-                          <mat-icon>{{(postsService.theme.key === 'default') ? 'brightness_5' : 'brightness_2'}}</mat-icon>
-                          <span i18n="Dark mode toggle label">Dark</span>
-                          <mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
-                        </button>
-                        <!-- <button *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('settings')))" (click)="openSettingsDialog()" mat-menu-item>
-                          <mat-icon>settings</mat-icon>
-                          <span i18n="Settings menu label">Settings</span>
-                        </button> -->
-                        <button (click)="openAboutDialog()" mat-menu-item>
-                            <mat-icon>info</mat-icon>
-                            <span i18n="About menu label">About</span>
-                        </button>
-                    </mat-menu>
                 </div>
             </div>
         </mat-toolbar>
@@ -44,14 +48,14 @@
                     <a *ngIf="postsService.config && postsService.config.Advanced.multi_user_mode && !postsService.isLoggedIn" mat-list-item (click)="sidenav.close()" routerLink='/login'><ng-container i18n="Navigation menu Login Page title">Login</ng-container></a>
                     <a *ngIf="postsService.config && allowSubscriptions && postsService.hasPermission('subscriptions')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a>
                     <a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a>
-                    <a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/tasks'><ng-container i18n="Navigation menu Tasks Page title">Tasks</ng-container></a>
+                    <a *ngIf="postsService.config && postsService.hasPermission('tasks_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/tasks'><ng-container i18n="Navigation menu Tasks Page title">Tasks</ng-container></a>
                     <ng-container *ngIf="postsService.config && postsService.hasPermission('settings')">
                         <mat-divider></mat-divider>
                         <a mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/settings'><ng-container i18n="Settings menu label">Settings</ng-container></a>
                     </ng-container>
                     <ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
                         <mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
-                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
+                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.display]="'inline-block'" [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
                     </ng-container>
                 </mat-nav-list>
             </mat-sidenav>

src/app/app.component.ts

@@ -20,6 +20,8 @@ import { SettingsComponent } from './settings/settings.component';
 import { AboutDialogComponent } from './dialogs/about-dialog/about-dialog.component';
 import { UserProfileDialogComponent } from './dialogs/user-profile-dialog/user-profile-dialog.component';
 import { SetDefaultAdminDialogComponent } from './dialogs/set-default-admin-dialog/set-default-admin-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 @Component({
   selector: 'app-root',
@@ -45,9 +47,12 @@ export class AppComponent implements OnInit, AfterViewInit {
   enableDownloadsManager = false;
 
   @ViewChild('sidenav') sidenav: MatSidenav;
+  @ViewChild('notifications') notifications: NotificationsComponent;
   @ViewChild('hamburgerMenu', { read: ElementRef }) hamburgerMenuButton: ElementRef;
   navigator: string = null;
 
+  notification_count = 0;
+
   constructor(public postsService: PostsService, public snackBar: MatSnackBar, private dialog: MatDialog,
     public router: Router, public overlayContainer: OverlayContainer, private elementRef: ElementRef) {
 
@@ -71,7 +76,7 @@ export class AppComponent implements OnInit, AfterViewInit {
 
   }
 
-  ngOnInit() {
+  ngOnInit(): void {
     if (localStorage.getItem('theme')) {
       this.setTheme(localStorage.getItem('theme'));
     }
@@ -90,15 +95,15 @@ export class AppComponent implements OnInit, AfterViewInit {
     });
   }
 
-  ngAfterViewInit() {
+  ngAfterViewInit(): void {
     this.postsService.sidenav = this.sidenav;
   }
 
-  toggleSidenav() {
+  toggleSidenav(): void {
     this.sidenav.toggle();
   }
 
-  loadConfig() {
+  loadConfig(): void {
     // loading config
     this.topBarTitle = this.postsService.config['Extra']['title_top'];
     const themingExists = this.postsService.config['Themes'];
@@ -164,7 +169,7 @@ export class AppComponent implements OnInit, AfterViewInit {
     this.componentCssClass = theme;
   }
 
-  flipTheme() {
+  flipTheme(): void {
     if (this.postsService.theme.key === 'default') {
       this.setTheme('dark');
     } else if (this.postsService.theme.key === 'dark') {
@@ -172,17 +177,12 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  themeMenuItemClicked(event) {
+  themeMenuItemClicked(event): void {
     this.flipTheme();
     event.stopPropagation();
   }
 
-  getSubscriptions() {
-
-  }
-
-
-  goBack() {
+  goBack(): void {
     if (!this.navigator) {
       this.router.navigate(['/home']);
     } else {
@@ -190,23 +190,41 @@ export class AppComponent implements OnInit, AfterViewInit {
     }
   }
 
-  openSettingsDialog() {
-    const dialogRef = this.dialog.open(SettingsComponent, {
+  openSettingsDialog(): void {
+    this.dialog.open(SettingsComponent, {
       width: '80vw'
     });
   }
 
-  openAboutDialog() {
-    const dialogRef = this.dialog.open(AboutDialogComponent, {
+  openAboutDialog(): void {
+    this.dialog.open(AboutDialogComponent, {
       width: '80vw'
     });
   }
 
-  openProfileDialog() {
-    const dialogRef = this.dialog.open(UserProfileDialogComponent, {
+  openProfileDialog(): void {
+    this.dialog.open(UserProfileDialogComponent, {
       width: '60vw'
     });
   }
 
+  openArchivesDialog(): void {
+    this.dialog.open(ArchiveViewerComponent, {
+      width: '85vw'
+    });
+  }
+
+  notificationCountUpdate(new_count: number): void {
+    this.notification_count = new_count;
+  }
+
+  notificationMenuOpened(): void {
+    this.notifications.getNotifications();
+  }
+
+  notificationMenuClosed(): void {
+    this.notifications.setNotificationsToRead();
+  }
+
 }
 

src/app/app.module.ts

@@ -29,6 +29,8 @@ import { MatPaginatorModule } from '@angular/material/paginator';
 import { MatSortModule } from '@angular/material/sort';
 import { MatTableModule } from '@angular/material/table';
 import { MatDatepickerModule } from '@angular/material/datepicker';
+import { MatChipsModule } from '@angular/material/chips';
+import { MatBadgeModule } from '@angular/material/badge';
 import { DragDropModule } from '@angular/cdk/drag-drop';
 import { ClipboardModule } from '@angular/cdk/clipboard';
 import { TextFieldModule } from '@angular/cdk/text-field';
@@ -49,10 +51,8 @@ import { CreatePlaylistComponent } from './create-playlist/create-playlist.compo
 import { SubscriptionsComponent } from './subscriptions/subscriptions.component';
 import { SubscribeDialogComponent } from './dialogs/subscribe-dialog/subscribe-dialog.component';
 import { SubscriptionComponent } from './subscription//subscription/subscription.component';
-import { SubscriptionFileCardComponent } from './subscription/subscription-file-card/subscription-file-card.component';
 import { SubscriptionInfoDialogComponent } from './dialogs/subscription-info-dialog/subscription-info-dialog.component';
 import { SettingsComponent } from './settings/settings.component';
-import { MatChipsModule } from '@angular/material/chips';
 import { NgxFileDropModule } from 'ngx-file-drop';
 import { AvatarModule } from 'ngx-avatars';
 import { ContentLoaderModule } from '@ngneat/content-loader';
@@ -74,7 +74,6 @@ import { ManageUserComponent } from './components/manage-user/manage-user.compon
 import { ManageRoleComponent } from './components/manage-role/manage-role.component';
 import { CookiesUploaderDialogComponent } from './dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component';
 import { LogsViewerComponent } from './components/logs-viewer/logs-viewer.component';
-import { ModifyPlaylistComponent } from './dialogs/modify-playlist/modify-playlist.component';
 import { ConfirmDialogComponent } from './dialogs/confirm-dialog/confirm-dialog.component';
 import { UnifiedFileCardComponent } from './components/unified-file-card/unified-file-card.component';
 import { RecentVideosComponent } from './components/recent-videos/recent-videos.component';
@@ -89,6 +88,13 @@ import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-butto
 import { TasksComponent } from './components/tasks/tasks.component';
 import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
 import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component';
+import { NotificationsComponent } from './components/notifications/notifications.component';
+import { NotificationsListComponent } from './components/notifications-list/notifications-list.component';
+import { TaskSettingsComponent } from './components/task-settings/task-settings.component';
+import { GenerateRssUrlComponent } from './dialogs/generate-rss-url/generate-rss-url.component';
+import { SortPropertyComponent } from './components/sort-property/sort-property.component';
+import { OnlyNumberDirective } from './directives/only-number.directive';
+import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component';
 
 registerLocaleData(es, 'es');
 
@@ -102,7 +108,6 @@ registerLocaleData(es, 'es');
         SubscriptionsComponent,
         SubscribeDialogComponent,
         SubscriptionComponent,
-        SubscriptionFileCardComponent,
         SubscriptionInfoDialogComponent,
         SettingsComponent,
         AboutDialogComponent,
@@ -123,7 +128,6 @@ registerLocaleData(es, 'es');
         ManageRoleComponent,
         CookiesUploaderDialogComponent,
         LogsViewerComponent,
-        ModifyPlaylistComponent,
         ConfirmDialogComponent,
         UnifiedFileCardComponent,
         RecentVideosComponent,
@@ -136,7 +140,14 @@ registerLocaleData(es, 'es');
         SkipAdButtonComponent,
         TasksComponent,
         UpdateTaskScheduleDialogComponent,
-        RestoreDbDialogComponent
+        RestoreDbDialogComponent,
+        NotificationsComponent,
+        NotificationsListComponent,
+        TaskSettingsComponent,
+        GenerateRssUrlComponent,
+        SortPropertyComponent,
+        OnlyNumberDirective,
+        ArchiveViewerComponent
     ],
     imports: [
         CommonModule,
@@ -174,6 +185,7 @@ registerLocaleData(es, 'es');
         MatTableModule,
         MatDatepickerModule,
         MatChipsModule,
+        MatBadgeModule,
         DragDropModule,
         ClipboardModule,
         TextFieldModule,

src/app/components/archive-viewer/archive-viewer.component.html

@@ -0,0 +1,143 @@
+<mat-form-field class="filter">
+  <mat-icon matPrefix>search</mat-icon>
+  <mat-label i18n="Filter">Filter</mat-label>
+  <input matInput [(ngModel)]="text_filter" (keyup)="applyFilter($event)" #input>
+</mat-form-field>
+
+<div [hidden]="!(archives && archives.length > 0)">
+    <div class="mat-elevation-z8">
+        <mat-table matSort [dataSource]="dataSource">
+
+          <!-- Select Column -->
+            <!-- Checkbox Column -->
+            <ng-container matColumnDef="select">
+              <mat-header-cell *matHeaderCellDef>
+                <mat-checkbox (change)="$event ? toggleAllRows() : null"
+                              [checked]="selection.hasValue() && isAllSelected()"
+                              [indeterminate]="selection.hasValue() && !isAllSelected()">
+                </mat-checkbox>
+              </mat-header-cell>
+              <mat-cell *matCellDef="let row">
+                <mat-checkbox (click)="$event.stopPropagation()"
+                              (change)="$event ? selection.toggle(row) : null"
+                              [checked]="selection.isSelected(row)">
+                </mat-checkbox>
+                <mat-icon class="audio-video-icon">{{(row.type === 'audio') ? 'audiotrack' : 'movie'}}</mat-icon>
+              </mat-cell>
+            </ng-container>
+      
+          <!-- Date Column -->
+          <ng-container matColumnDef="timestamp">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Date">Date</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element"> {{element.timestamp*1000 | date: 'short'}} </mat-cell>
+          </ng-container>
+      
+          <!-- Title Column -->
+          <ng-container matColumnDef="title">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="max-two-lines" [matTooltip]="element.title ? element.title : null">
+                    {{element.title}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+           <!-- ID Column -->
+           <ng-container matColumnDef="id">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="ID">ID</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.title ? element.title : null">
+                    {{element.id}}
+                </span>
+            </mat-cell>
+          </ng-container>
+
+          <!-- Extractor Column -->
+          <ng-container matColumnDef="extractor">
+            <mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Extractor">Extractor</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+                <span class="one-line" [matTooltip]="element.extractor? element.extractor : null">
+                    {{element.extractor}}
+                </span>
+            </mat-cell>
+          </ng-container>
+      
+            <mat-header-row *matHeaderRowDef="displayedColumns; sticky: true"></mat-header-row>
+            <mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
+        </mat-table>
+      </div>
+</div>
+
+<div *ngIf="(!archives || archives.length === 0)">
+  <h4 style="text-align: center; margin-top: 10px;" i18n="Archives empty">Archives empty</h4>
+</div>
+
+<div style="margin: 10px 10px 10px 0px; display: flex;">
+  <span style="flex-grow: 1;" class="flex-items">
+    <button [disabled]="selection.selected.length === 0" color="warn" style="margin: 10px;" mat-stroked-button i18n="Delete selected" (click)="openDeleteSelectedArchivesDialog()">Delete selected</button>
+  </span>
+  <span class="flex-items">
+    <button [disabled]="!(archives && archives.length > 0)" (click)="downloadArchive()" mat-stroked-button i18n="Download archive">Download archive</button>
+    <mat-form-field style="width: 150px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="Subscription">Subscription</mat-label>
+      <mat-select [ngModel]="sub_id" (ngModelChange)="subFilterSelectionChanged($event)">
+        <mat-option [value]="'none'" i18n="None">None</mat-option>
+        <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+      </mat-select>
+    </mat-form-field>
+    <mat-form-field style="width: 100px; margin-bottom: -1.25em; margin-left: 10px;">
+      <mat-label i18n="File type">File type</mat-label>
+      <mat-select [ngModel]="type" (ngModelChange)="typeFilterSelectionChanged($event)" [disabled]="sub_id !== 'none'">
+        <mat-option [value]="'both'" i18n="Both">Both</mat-option>
+        <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+        <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+      </mat-select>
+    </mat-form-field>
+  </span>
+</div>
+
+<div class="file-drop-parent">
+  <ngx-file-drop [multiple]="false" accept=".txt" dropZoneLabel="Drop file here" (onFileDrop)="dropped($event)">
+      <ng-template class="file-drop" ngx-file-drop-content-tmp let-openFileSelector="openFileSelector">
+          <div style="text-align: center">
+              <div>
+                  <ng-container i18n="Drag and Drop">Drag and Drop</ng-container>
+              </div>
+              <div style="margin-top: 6px;">
+                  <button mat-stroked-button (click)="openFileSelector()">Browse Files</button>
+              </div>
+          </div>
+      </ng-template>
+  </ngx-file-drop>
+</div>
+
+<div style="margin-top: 10px; color: white">
+  <table class="table">
+      <tbody class="upload-name-style">
+          <tr *ngFor="let item of files; let i=index">
+              <td style="vertical-align: middle; border-top: unset">
+                  <strong>{{ item.relativePath }}</strong>
+              </td>
+              <td style="border-top: unset">
+                <div style="float: right">
+                  <mat-form-field style="width: 150px;">
+                    <mat-label i18n="Subscription">Subscription</mat-label>
+                    <mat-select [ngModel]="upload_sub_id" (ngModelChange)="subUploadFilterSelectionChanged($event)">
+                      <mat-option [value]="'none'" i18n="None">None</mat-option>
+                      <mat-option *ngFor="let sub of postsService.subscriptions" [value]="sub.id">{{sub.name}}</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <mat-form-field style="width: 100px; margin-left: 10px">
+                    <mat-label i18n="File type">File type</mat-label>
+                    <mat-select [(ngModel)]="upload_type" [value]="upload_type" [disabled]="upload_sub_id !== 'none'">
+                      <mat-option [value]="'video'" i18n="Video">Video</mat-option>
+                      <mat-option [value]="'audio'" i18n="Audio">Audio</mat-option>
+                    </mat-select>
+                  </mat-form-field>
+                  <button style="margin-left: 10px" [disabled]="uploading_archive || uploaded_archive" (click)="importArchive()" matTooltip="Upload" i18n-matTooltip="Upload" mat-mini-fab><mat-icon>publish</mat-icon><mat-spinner *ngIf="uploading_archive" class="spinner" [diameter]="38"></mat-spinner></button>
+                </div>
+              </td>
+          </tr>
+      </tbody>
+  </table>
+</div>

src/app/components/archive-viewer/archive-viewer.component.scss

@@ -0,0 +1,42 @@
+.filter {
+    width: 100%;
+}
+
+.spinner {
+    bottom: 1px;
+    left: 0.5px;
+    position: absolute;
+}
+
+.mat-mdc-table {
+    width: 100%;
+    max-height: 60vh;
+    overflow: auto;
+}
+
+.max-two-lines {
+    display: -webkit-box;
+    display: -moz-box;
+    max-height: 2.4em;
+    line-height: 1.2em;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    -webkit-box-orient: vertical;
+    -webkit-line-clamp: 2;
+}
+
+:host ::ng-deep .ngx-file-drop__content {
+    width: 100%;
+    top: -12px;
+    position: relative;
+}
+
+.file-drop-parent {
+    padding: 0px 10px 0px 10px;
+}
+
+.flex-items {
+    display: flex;
+    align-items: center;
+}
+

src/app/components/archive-viewer/archive-viewer.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ArchiveViewerComponent } from './archive-viewer.component';
+
+describe('ArchiveViewerComponent', () => {
+  let component: ArchiveViewerComponent;
+  let fixture: ComponentFixture<ArchiveViewerComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ArchiveViewerComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(ArchiveViewerComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src/app/components/archive-viewer/archive-viewer.component.ts

@@ -0,0 +1,198 @@
+import { SelectionModel } from '@angular/cdk/collections';
+import { Component, ViewChild } from '@angular/core';
+import { MatDialog } from '@angular/material/dialog';
+import { MatSort } from '@angular/material/sort';
+import { MatTableDataSource } from '@angular/material/table';
+import { FileType } from 'api-types';
+import { Archive } from 'api-types/models/Archive';
+import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
+import { PostsService } from 'app/posts.services';
+import { NgxFileDropEntry } from 'ngx-file-drop';
+
+@Component({
+  selector: 'app-archive-viewer',
+  templateUrl: './archive-viewer.component.html',
+  styleUrls: ['./archive-viewer.component.scss']
+})
+export class ArchiveViewerComponent {
+  // table
+  displayedColumns: string[] = ['select', 'timestamp', 'title', 'id', 'extractor'];
+  dataSource = null;
+  selection = new SelectionModel<Archive>(true, []);
+
+  // general
+  archives = null;
+  archives_retrieved = false;
+  text_filter = '';
+  sub_id = 'none';
+  upload_sub_id = 'none';
+  type: FileType | 'both' = 'both';
+  upload_type: FileType = FileType.VIDEO;
+
+  // importing
+  uploading_archive = false;
+  uploaded_archive = false;
+  files = [];
+
+  typeSelectOptions = {
+    video: {
+      key: 'video',
+      label: $localize`Video`
+    },
+    audio: {
+      key: 'audio',
+      label: $localize`Audio`
+    }
+  };
+
+  @ViewChild(MatSort) sort: MatSort;
+  
+  constructor(public postsService: PostsService, private dialog: MatDialog) {
+
+  }
+
+  ngOnInit() {
+    this.getArchives();
+  }
+
+  applyFilter(event: Event) {
+    const filterValue = (event.target as HTMLInputElement).value;
+    this.dataSource.filter = filterValue.trim().toLowerCase();
+  }
+
+  /** Whether the number of selected elements matches the total number of rows. */
+  isAllSelected() {
+    const numSelected = this.selection.selected.length;
+    const numRows = this.dataSource.data.length;
+    return numSelected === numRows;
+  }
+
+  /** Selects all rows if they are not all selected; otherwise clear selection. */
+  toggleAllRows() {
+    if (this.isAllSelected()) {
+      this.selection.clear();
+      return;
+    }
+
+    this.selection.select(...this.dataSource.data);
+  }
+
+  typeFilterSelectionChanged(value): void {
+    this.type = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    this.getArchives();
+  }
+
+  subFilterSelectionChanged(value): void {
+    this.sub_id = value;
+    this.dataSource.filter = '';
+    this.text_filter = '';
+    if (this.sub_id !== 'none') {
+      this.type = this.postsService.getSubscriptionByID(this.sub_id)['type'];
+    }
+    this.getArchives();
+  }
+
+  subUploadFilterSelectionChanged(value): void {
+    this.upload_sub_id = value;
+    if (this.upload_sub_id !== 'none') {
+      this.upload_type = this.postsService.getSubscriptionByID(this.upload_sub_id)['type'];
+    }
+  }
+
+  getArchives(): void {
+    this.postsService.getArchives(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      if (res['archives'] !== null 
+        && res['archives'] !== undefined
+        && JSON.stringify(this.archives) !== JSON.stringify(res['archives'])) {
+          this.archives = res['archives']
+          this.dataSource = new MatTableDataSource<Archive>(this.archives);
+          this.dataSource.sort = this.sort;
+      } else {
+        // failed to get downloads
+      }
+    });
+  }
+
+  importArchive(): void {
+    this.uploading_archive = true;
+        for (const droppedFile of this.files) {
+      // Is it a file?
+      if (droppedFile.fileEntry.isFile) {
+        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
+        fileEntry.file(async (file: File) => {          
+          const archive_base64 = await blobToBase64(file);
+          this.postsService.importArchive(archive_base64 as string, this.upload_type, this.upload_sub_id === 'none' ? null : this.upload_sub_id).subscribe(res => {
+            this.uploading_archive = false;
+            if (res['success']) {
+              this.uploaded_archive = true;
+              this.postsService.openSnackBar($localize`Archive successfully imported!`);
+            }
+            this.getArchives();
+          }, err => {
+            console.error(err);
+            this.uploading_archive = false;
+          });
+        });
+      }
+    }
+  }
+
+  downloadArchive(): void {
+    this.postsService.downloadArchive(this.type === 'both' ? null : this.type, this.sub_id === 'none' ? null : this.sub_id).subscribe(res => {
+      const blob: Blob = res;
+      saveAs(blob, 'archive.txt');
+    });
+  }
+
+  openDeleteSelectedArchivesDialog(): void {
+    const dialogRef = this.dialog.open(ConfirmDialogComponent, {
+      data: {
+        dialogTitle: $localize`Delete archives`,
+        dialogText: $localize`Would you like to delete ${this.selection.selected.length}:selected archives amount: archive(s)?`,
+        submitText: $localize`Delete`,
+        warnSubmitColor: true
+      }
+    });
+    dialogRef.afterClosed().subscribe(confirmed => {
+      if (confirmed) {
+        this.deleteSelectedArchives();
+      }
+    });
+    
+  }
+
+  deleteSelectedArchives(): void {
+    for (const archive of this.selection.selected) {
+      this.archives = this.archives.filter((_archive: Archive) => !(archive['extractor'] === _archive['extractor'] && archive['id'] !== _archive['id']));
+    }
+    this.postsService.deleteArchiveItems(this.selection.selected).subscribe(res => {
+      if (res['success']) {
+        this.postsService.openSnackBar($localize`Successfully deleted archive items!`);
+      } else {
+        this.postsService.openSnackBar($localize`Failed to delete archive items!`);
+      }
+      this.getArchives();
+    });
+    this.selection.clear();
+  }
+
+  public dropped(files: NgxFileDropEntry[]) {
+    this.files = files;
+    this.uploading_archive = false;
+    this.uploaded_archive = false;
+  }
+
+  originalOrder = (): number => {
+    return 0;
+  }
+}
+
+function blobToBase64(blob: Blob) {
+  return new Promise((resolve, _) => {
+    const reader = new FileReader();
+    reader.onloadend = () => resolve(reader.result);
+    reader.readAsDataURL(blob);
+  });
+}

src/app/components/custom-playlists/custom-playlists.component.html

@@ -2,7 +2,7 @@
     <div class="container">
         <div class="row justify-content-center">
             <div *ngFor="let playlist of playlists; let i = index" class="mb-2 mt-2" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
-                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [loading]="false"></app-unified-file-card>
+                <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? this.postsService.token : ''" [loading]="false"></app-unified-file-card>
             </div>
         </div>
     </div>

src/app/components/custom-playlists/custom-playlists.component.ts

@@ -3,7 +3,7 @@ import { PostsService } from 'app/posts.services';
 import { Router } from '@angular/router';
 import { MatDialog } from '@angular/material/dialog';
 import { CreatePlaylistComponent } from 'app/create-playlist/create-playlist.component';
-import { ModifyPlaylistComponent } from 'app/dialogs/modify-playlist/modify-playlist.component';
+import { Playlist } from 'api-types';
 
 @Component({
   selector: 'app-custom-playlists',
@@ -32,7 +32,7 @@ export class CustomPlaylistsComponent implements OnInit {
     });
   }
 
-  getAllPlaylists() {
+  getAllPlaylists(): void {
     this.playlists_received = false;
     // must call getAllFiles as we need to get category playlists as well
     this.postsService.getPlaylists(true).subscribe(res => {
@@ -42,22 +42,25 @@ export class CustomPlaylistsComponent implements OnInit {
   }
 
   // creating a playlist
-  openCreatePlaylistDialog() {
+  openCreatePlaylistDialog(): void {
     const dialogRef = this.dialog.open(CreatePlaylistComponent, {
       data: {
-      }
+        create_mode: true
+      },
+      minWidth: '90vw',
+      minHeight: '95vh'
     });
     dialogRef.afterClosed().subscribe(result => {
       if (result) {
         this.getAllPlaylists();
-        this.postsService.openSnackBar('Successfully created playlist!', '');
+        this.postsService.openSnackBar($localize`Successfully created playlist!`);
       } else if (result === false) {
-        this.postsService.openSnackBar('ERROR: failed to create playlist!', '');
+        this.postsService.openSnackBar($localize`ERROR: failed to create playlist!`);
       }
     });
   }
 
-  goToPlaylist(info_obj) {
+  goToPlaylist(info_obj: { file: Playlist; }): void {
     const playlist = info_obj.file;
     const playlistID = playlist.id;
 
@@ -72,11 +75,12 @@ export class CustomPlaylistsComponent implements OnInit {
       }
     } else {
       // playlist not found
+      // TODO: Make translatable
       console.error(`Playlist with ID ${playlistID} not found!`);
     }
   }
 
-  downloadPlaylist(playlist_id, playlist_name) {
+  downloadPlaylist(playlist_id: string, playlist_name: string): void {
     this.downloading_content[playlist_id] = true;
     this.postsService.downloadPlaylistFromServer(playlist_id).subscribe(res => {
       this.downloading_content[playlist_id] = false;
@@ -86,33 +90,34 @@ export class CustomPlaylistsComponent implements OnInit {
 
   }
 
-  deletePlaylist(args) {
+  deletePlaylist(args: { file: Playlist; index: number; }): void {
     const playlist = args.file;
     const index = args.index;
     const playlistID = playlist.id;
-    this.postsService.removePlaylist(playlistID, playlist.type).subscribe(res => {
+    this.postsService.removePlaylist(playlistID).subscribe(res => {
       if (res['success']) {
         this.playlists.splice(index, 1);
-        this.postsService.openSnackBar('Playlist successfully removed.', '');
+        this.postsService.openSnackBar($localize`Playlist successfully removed.`);
       }
       this.getAllPlaylists();
     });
   }
 
-  editPlaylistDialog(args) {
+  editPlaylistDialog(args: { playlist: Playlist; index: number; }): void {
     const playlist = args.playlist;
     const index = args.index;
-    const dialogRef = this.dialog.open(ModifyPlaylistComponent, {
+    const dialogRef = this.dialog.open(CreatePlaylistComponent, {
       data: {
         playlist_id: playlist.id,
-        width: '65vw'
-      }
+        create_mode: false
+      },
+      minWidth: '85vw'
     });
 
-    dialogRef.afterClosed().subscribe(res => {
+    dialogRef.afterClosed().subscribe(() => {
       // updates playlist in file manager if it changed
       if (dialogRef.componentInstance.playlist_updated) {
-        this.playlists[index] = dialogRef.componentInstance.original_playlist;
+        this.playlists[index] = dialogRef.componentInstance.playlist;
       }
     });
   }

src/app/components/downloads/downloads.component.ts

@@ -8,6 +8,7 @@ import { MatDialog } from '@angular/material/dialog';
 import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
 import { MatSort } from '@angular/material/sort';
 import { Clipboard } from '@angular/cdk/clipboard';
+import { Download } from 'api-types';
 
 @Component({
   selector: 'app-downloads',
@@ -40,7 +41,7 @@ import { Clipboard } from '@angular/cdk/clipboard';
 })
 export class DownloadsComponent implements OnInit, OnDestroy {
 
-  @Input() uids = null;
+  @Input() uids: string[] = null;
 
   downloads_check_interval = 1000;
   downloads = [];
@@ -68,7 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   @ViewChild(MatPaginator) paginator: MatPaginator;
   @ViewChild(MatSort) sort: MatSort;
 
-  sort_downloads = (a, b) => {
+  sort_downloads = (a: Download, b: Download): number => {
     const result = b.timestamp_start - a.timestamp_start;
     return result;
   }
@@ -166,7 +167,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   pauseDownload(download_uid: string): void {
     this.postsService.pauseDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause download! See server logs for more info.`);
       }
     });
   }
@@ -174,7 +175,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   pauseAllDownloads(): void {
     this.postsService.pauseAllDownloads().subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause all downloads! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause all downloads! See server logs for more info.`);
       }
     });
   }
@@ -182,7 +183,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   resumeDownload(download_uid: string): void {
     this.postsService.resumeDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to resume download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to resume download! See server logs for more info.`);
       }
     });
   }
@@ -190,7 +191,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   resumeAllDownloads(): void {
     this.postsService.resumeAllDownloads().subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to resume all downloads! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to resume all downloads! See server logs for more info.`);
       }
     });
   }
@@ -198,7 +199,11 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   restartDownload(download_uid: string): void {
     this.postsService.restartDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to restart download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to restart download! See server logs for more info.`);
+      } else {
+        if (this.uids && res['new_download_uid']) {
+          this.uids.push(res['new_download_uid']);
+        }
       }
     });
   }
@@ -206,7 +211,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   cancelDownload(download_uid: string): void {
     this.postsService.cancelDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to cancel download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to cancel download! See server logs for more info.`);
       }
     });
   }
@@ -214,12 +219,12 @@ export class DownloadsComponent implements OnInit, OnDestroy {
   clearDownload(download_uid: string): void {
     this.postsService.clearDownload(download_uid).subscribe(res => {
       if (!res['success']) {
-        this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
+        this.postsService.openSnackBar($localize`Failed to pause download! See server logs for more info.`);
       }
     });
   }
 
-  watchContent(download): void {
+  watchContent(download: Download): void {
     const container = download['container'];
     localStorage.setItem('player_navigator', this.router.url.split(';')[0]);
     const is_playlist = container['uids']; // hacky, TODO: fix
@@ -230,7 +235,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     }
   }
 
-  combineDownloads(downloads_old, downloads_new) {
+  combineDownloads(downloads_old: Download[], downloads_new: Download[]): Download[] {
     // only keeps downloads that exist in the new set
     downloads_old = downloads_old.filter(download_old => downloads_new.some(download_new => download_new.uid === download_old.uid));
 
@@ -251,7 +256,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     return downloads_old;
   }
 
-  showError(download) {
+  showError(download: Download): void {
     const copyToClipboardEmitter = new EventEmitter<boolean>();
     this.dialog.open(ConfirmDialogComponent, {
       data: {
@@ -272,10 +277,3 @@ export class DownloadsComponent implements OnInit, OnDestroy {
     });
   }
 }
-
-export interface Download {
-  timestamp_start: number;
-  title: string;
-  step_index: number;
-  progress: string;
-}

src/app/components/login/login.component.html

@@ -1,31 +1,36 @@
 <mat-card class="login-card">
-    <mat-tab-group style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
+    <mat-tab-group mat-stretch-tabs style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
         <mat-tab label="Login" i18n-label="Login">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="loginUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="loginUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>
         <mat-tab *ngIf="registrationEnabled" label="Register" i18n-label="Register">
             <div style="margin-top: 10px;">
-                <mat-form-field>
-                    <input [(ngModel)]="registrationUsernameInput" matInput placeholder="User name" i18n-placeholder="User name">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="User name">User name</mat-label>
+                    <input [(ngModel)]="registrationUsernameInput" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput placeholder="Password" i18n-placeholder="Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Password">Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
             <div>
-                <mat-form-field>
-                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput placeholder="Confirm Password" i18n-placeholder="Confirm Password">
+                <mat-form-field style="width: 100%">
+                    <mat-label i18n="Confirm Password">Confirm Password</mat-label>
+                    <input [(ngModel)]="registrationPasswordConfirmationInput" (click)="register()" type="password" matInput>
                 </mat-form-field>
             </div>
         </mat-tab>

src/app/components/login/login.component.scss

@@ -12,17 +12,15 @@
 }
 
 .login-button-div {
-    margin-bottom: 10px;
-    margin-top: 10px;
-    margin-left: -16px;
-    margin-right: -16px;
     bottom: 0px;
     width: 100%;
+    height: 42px;
     position: absolute;
 }
 
 .login-button-div > button {
     width: 100%;
+    height: 100%;
     border-radius: 0px 0px 4px 4px !important;
 }
 

src/app/components/logs-viewer/logs-viewer.component.ts

@@ -43,17 +43,17 @@ export class LogsViewerComponent implements OnInit {
           })
         });
       } else {
-        this.postsService.openSnackBar('Failed to retrieve logs!');
+        this.postsService.openSnackBar($localize`Failed to retrieve logs!`);
       }
     }, err => {
       this.logs_loading = false;
       console.error(err);
-      this.postsService.openSnackBar('Failed to retrieve logs!');
+      this.postsService.openSnackBar($localize`Failed to retrieve logs!`);
     });
   }
 
   copiedLogsToClipboard() {
-    this.postsService.openSnackBar('Logs copied to clipboard!');
+    this.postsService.openSnackBar($localize`Logs copied to clipboard!`);
   }
 
   clearLogs() {
@@ -72,12 +72,12 @@ export class LogsViewerComponent implements OnInit {
             this.logs = [];
             this.logs_text = '';
             this.getLogs();
-            this.postsService.openSnackBar('Logs successfully cleared!');
+            this.postsService.openSnackBar($localize`Logs successfully cleared!`);
           } else {
-            this.postsService.openSnackBar('Failed to clear logs!');
+            this.postsService.openSnackBar($localize`Failed to clear logs!`);
           }
         }, err => {
-          this.postsService.openSnackBar('Failed to clear logs!');
+          this.postsService.openSnackBar($localize`Failed to clear logs!`);
         });
       }
     });

src/app/components/manage-role/manage-role.component.html

@@ -1,17 +1,15 @@
-<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.name}}</h4>
+<h4 *ngIf="role" mat-dialog-title><ng-container i18n="Manage role dialog title">Manage role</ng-container>&nbsp;-&nbsp;{{role.key}}</h4>
 
 <mat-dialog-content *ngIf="role">
-    <mat-list>
-        <mat-list-item role="listitem" *ngFor="let permission of available_permissions">
-            <h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
-            <span matLine>
-                <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
-                    <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
-                    <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
-                </mat-radio-group>
-            </span>
-        </mat-list-item>
-    </mat-list>
+    <div *ngFor="let permission of available_permissions">
+        <div matListItemTitle>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</div>
+        <div matListItemLine>
+            <mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission)" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
+                <mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
+                <mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
+            </mat-radio-group>
+        </div>
+    </div>
 </mat-dialog-content>
 
 <mat-dialog-actions>

src/app/components/manage-role/manage-role.component.scss

@@ -1,4 +0,0 @@
-.mat-radio-button {
-    margin-right: 10px;
-    margin-top: 5px;
-}

src/app/components/manage-role/manage-role.component.ts

@@ -14,16 +14,17 @@ export class ManageRoleComponent implements OnInit {
   permissions = null;
 
   permissionToLabel = {
-    'filemanager': 'File manager',
-    'settings': 'Settings access',
-    'subscriptions': 'Subscriptions',
-    'sharing': 'Share files',
-    'advanced_download': 'Use advanced download mode',
-    'downloads_manager': 'Use downloads manager'
+    'filemanager': $localize`File manager`,
+    'settings': $localize`Settings access`,
+    'subscriptions': $localize`Subscriptions`,
+    'sharing': $localize`Share files`,
+    'advanced_download': $localize`Use advanced download mode`,
+    'downloads_manager': $localize`Use downloads manager`,
+    'tasks_manager': $localize`Use tasks manager`,
   }
 
   constructor(public postsService: PostsService, private dialogRef: MatDialogRef<ManageRoleComponent>,
-              @Inject(MAT_DIALOG_DATA) public data: any) {
+              @Inject(MAT_DIALOG_DATA) public data: {role: string}) {
     if (this.data) {
       this.role = this.data.role;
       this.available_permissions = this.postsService.available_permissions;