Updated name of arm64 TwitchDownloaderCLI asset

Fixed issue that prevented TwitchDownloader from working in windows

.github/dependabot.yaml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/backend/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -15,9 +15,9 @@ jobs:
       - name: checkout code
         uses: actions/checkout@v2
       - name: setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
-          node-version: '12'
+          node-version: '16'
           cache: 'npm'
       - name: install dependencies
         run: |
@@ -33,7 +33,7 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@1.1.2
+        uses: jsdaniell/create-json@v1.2.2
         with:
           name: "version.json"
           json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -55,7 +55,7 @@ jobs:
           Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
       - name: upload build artifact
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: youtubedl-material
           path: build

.github/workflows/close-issue-if-noresponse.yml

@@ -0,0 +1,38 @@
+name: No Response
+
+# Both `issue_comment` and `scheduled` event types are required for this Action
+# to work properly.
+on:
+  issue_comment:
+    types: [created]
+  schedule:
+    # Schedule for five minutes after the hour, every hour
+    - cron: '5 * * * *'
+
+# By specifying the access of one of the scopes, all of those that are not
+# specified are set to 'none'.
+permissions:
+  issues: write
+
+jobs:
+  noResponse:
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'Tzahi12345/YoutubeDL-Material' }}
+    steps:
+      - uses: lee-dohm/no-response@v0.5.0
+        with:
+          token: ${{ github.token }}
+          # Comment to post when closing an Issue for lack of response. Set to `false` to disable
+          closeComment: >
+            This issue has been automatically closed because there has been no response
+            to our request for more information from the original author. With only the
+            information that is currently in the issue, we don't have enough information
+            to take action. Please reach out if you have or find the answers we need so
+            that we can investigate further. We will re-open this issue if you provide us
+            with the requested information with a comment under this issue.
+            Thank you for your understanding and for trying to help make this application
+            a better one!
+          # Number of days of inactivity before an issue is closed for lack of response.
+          daysUntilClose: 21
+          # Label requiring a response.
+          responseRequiredLabel: "💬 response-needed"

.github/workflows/docker-pr.yml

@@ -0,0 +1,27 @@
+name: docker-pr
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@v1.2.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: Build docker images
+        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker-release.yml

@@ -6,35 +6,75 @@ on:
       tags:
         description: 'Docker tags'
         required: true
+  release:
+    types: [published]
 
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+
       - name: Set hash
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
       - name: Get current date
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@1.1.2
+        uses: jsdaniell/create-json@v1.2.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
+
+      - name: Set image tag
+        id: tags
+        run: |
+          if [ "${{ github.event.inputs.tags }}" != "" ]; then
+            echo "::set-output name=tags::${{ github.event.inputs.tags }}"
+          elif [ ${{ github.event.action }} == "release" ]; then
+            echo "::set-output name=tags::${{ github.event.release.tag_name }}"
+          else
+            echo "Unknown workflow trigger: ${{ github.event.action }}! Cannot determine default tag."
+            exit 1
+          fi
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,value=${{ steps.tags.outputs.tags }}
+            type=raw,value=latest
+
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
+
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: build & push images
         uses: docker/build-push-action@v2
         with:
@@ -42,4 +82,5 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          tags: ${{ github.event.inputs.tags }}
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/docker.yml

@@ -3,6 +3,19 @@ name: docker
 on:
   push:
     branches: [master]
+    paths-ignore:
+      - '.github/**'
+      - '.vscode/**'
+      - 'chrome-extension/**'
+      - 'releases/**'
+      - '**/**.md'
+      - '**.crx'
+      - '**.pem'
+      - '.dockerignore'
+      - '.gitignore'
+  schedule:
+    - cron:  '34 4 * * 2'
+  workflow_dispatch:
 
 jobs:
   build-and-push:
@@ -10,28 +23,58 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+
       - name: Set hash
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
       - name: Get current date
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@1.1.2
+        uses: jsdaniell/create-json@v1.2.2
         with:
           name: "version.json"
-          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
+
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
+
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        # Defaults:
+        #     DOCKERHUB_USERNAME  : tzahi12345
+        #     DOCKERHUB_REPO      : youtubedl-material
+        #     DOCKERHUB_MASTER_TAG: nightly
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}-{{ date 'YYYY-MM-DD' }}
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}
+            type=sha,prefix=sha-,format=short
+
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: build & push images
         uses: docker/build-push-action@v2
         with:
@@ -39,4 +82,5 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          tags: tzahi12345/youtubedl-material:nightly
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+    "recommendations": [
+        "angular.ng-template",
+        "dbaeumer.vscode-eslint",
+        "waderyan.gitblame",
+        "42crunch.vscode-openapi",
+        "redhat.vscode-yaml",
+        "christian-kohler.npm-intellisense",
+        "hbenl.vscode-mocha-test-adapter"
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "mochaExplorer.files": "backend/test/**/*.js",
+    "mochaExplorer.cwd": "backend",
+    "mochaExplorer.globImplementation": "vscode",
+    "mochaExplorer.env": {
+        "YTDL_MODE": "debug"
+    }
+}

.vscode/tasks.json

@@ -1,25 +1,60 @@
 {
 	"version": "2.0.0",
+	"windows": {
+        "options": {
+            "shell": {
+                "executable": "cmd.exe",
+                "args": [
+                    "/d", "/c"
+                ]
+            }
+        }
+    },
 	"tasks": [
 		{
 			"type": "npm",
 			"script": "start",
 			"problemMatcher": [],
 			"label": "Dev: start frontend",
-			"detail": "ng serve"
+			"detail": "ng serve",
+			"presentation": {
+				"echo": true,
+				"reveal": "always",
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
+			}
 		},
 		{
 			"label": "Dev: start backend",
 			"type": "shell",
-			"command": "set YTDL_MODE=debug && node app.js",
+			"command": "node app.js",
 			"options": {
-				"cwd": "./backend"
+				"cwd": "./backend",
+				"env": {
+					"YTDL_MODE": "debug"
+				}
 			},
 			"presentation": {
+				"echo": true,
 				"reveal": "always",
-				"panel": "new"
+				"focus": true,
+				"panel": "shared",
+				"showReuseMessage": true,
+				"clear": false
 			},
-			"problemMatcher": []
+			"problemMatcher": [],
+			"dependsOn": ["Dev: post-build"]
+		},
+		{
+			"label": "Dev: post-build",
+			"type": "shell",
+			"command": "node src/postbuild.mjs"
+		},
+		{
+			"label": "Dev: run all",
+			"dependsOn": ["Dev: start backend", "Dev: start frontend"]
 		}
 	]
 }

Dockerfile

@@ -1,54 +1,84 @@
-FROM alpine:latest AS ffmpeg
+# Fetching our ffmpeg
+FROM ubuntu:22.04 AS ffmpeg
+ENV DEBIAN_FRONTEND=noninteractive
+# Use script due local build compability
+COPY docker-utils/ffmpeg-fetch.sh .
+RUN chmod +x ffmpeg-fetch.sh
+RUN sh ./ffmpeg-fetch.sh
 
-COPY docker-build.sh .
-RUN sh ./docker-build.sh
-
-FROM alpine:latest as frontend
-
-RUN apk add --no-cache \
-  npm
-
-RUN npm install -g @angular/cli
-
-WORKDIR /build
-COPY [ "package.json", "package-lock.json", "/build/" ]
-RUN npm install
-
-COPY [ "angular.json", "tsconfig.json", "/build/" ]
-COPY [ "src/", "/build/src/" ]
-RUN npm run build
-
-#--------------#
-
-FROM alpine:latest
-
-ENV UID=1000 \
-  GID=1000 \
-  USER=youtube
 
+# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
+# Go to 20.04
+FROM ubuntu:20.04 AS base
+ARG DEBIAN_FRONTEND=noninteractive
+ENV UID=1000
+ENV GID=1000
+ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
-
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
-
-RUN apk add --no-cache \
-  npm \
-  python2 \
-  python3 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
-
-WORKDIR /app
-COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
-COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
-COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
 ENV PM2_HOME=/app/pm2
-RUN npm install pm2 -g
+ENV ALLOW_CONFIG_MUTATIONS=true
-RUN npm install && chown -R $UID:$GID ./
+RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
+    apt update && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt install -y --no-install-recommends nodejs && \
+    npm -g install npm n && \
+    n 16.14.2 && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
 
+
+# Build frontend
+FROM base as frontend
+RUN npm install -g @angular/cli
+WORKDIR /build
+COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
+COPY [ "src/", "/build/src/" ]
+RUN npm install && \
+    npm run build && \
+    ls -al /build/backend/public
+RUN npm uninstall -g @angular/cli
+RUN rm -rf node_modules
+
+
+# Install backend deps
+FROM base as backend
+WORKDIR /app
+COPY [ "backend/","/app/" ]
+RUN npm config set strict-ssl false && \
+    npm install --prod && \
+    ls -al
+
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
+
+# Final image
+FROM base
+RUN npm install -g pm2 && \
+    apt update && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install pycryptodomex
+WORKDIR /app
+# User 1000 already exist from base image
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
+COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
+COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"]
+RUN chown $UID:$GID .
+RUN chmod +x /app/fix-scripts/*.sh
+# Add some persistence data
+#VOLUME ["/app/appdata"]
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime", "pm2.config.js" ]
+CMD [ "npm","start" ]

Dockerfile.heroku

@@ -0,0 +1,2 @@
+FROM tzahi12345/youtubedl-material:latest
+CMD [ "npm", "start" ]

Procfile

@@ -1 +0,0 @@
-web: npm start --prefix backend

Public API v1.yaml

@@ -16,7 +16,7 @@ paths:
         - downloader
       summary: Download video file
       description: |-
-        Downloads a video file with the given URL. Will include global args if they exist.
+        Downloads a file with the given URL. Will include global args if they exist.
 
 
         HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status.
@@ -41,7 +41,7 @@ paths:
     post:
       tags:
         - downloader
-      summary: Download video file
+      summary: Generates arguments used to download file
       description: Generates args, used for checking what args would run if you ran downloadFile
       operationId: post-generateArgs
       requestBody:
@@ -97,6 +97,11 @@ paths:
       summary: Get all files
       description: Gets all files and playlists stored in the db
       operationId: get-getAllFiles
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GetAllFilesRequest'
       responses:
         '200':
           description: OK
@@ -106,6 +111,37 @@ paths:
                 $ref: '#/components/schemas/GetAllFilesResponse'
       security:
         - Auth query parameter: []
+  /api/rss:
+    get:
+      tags:
+        - files
+      summary: Generates an RSS feed
+      description: Generates an RSS feed for downloaded files
+      operationId: get-rss
+      parameters:
+        - in: query
+          name: params
+          schema:
+            allOf:
+            - $ref: '#/components/schemas/GetAllFilesRequest'
+            - type: object
+              properties:
+                uuid:
+                  type: string
+                  description: user uid
+                  default: null
+          style: form
+          explode: true
+      responses:
+        '200':
+          description: OK
+          content:
+            text/plain:
+              schema:
+                type: string
+                description: RSS feed
+      security:
+        - Auth query parameter: []
   /api/getFile:
     post:
       tags:
@@ -129,6 +165,27 @@ paths:
           description: User is not authorized to view the file.
       security:
         - Auth query parameter: []
+  /api/updateFile:
+    post:
+      tags:
+        - files
+      summary: Updates file database object
+      description: Updates a file db object using its uid and a change object.
+      operationId: post-updateFile
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UpdateFileRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/enableSharing:
     post:
       tags:
@@ -521,6 +578,69 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
+  /api/deleteArchiveItems:
+    post:
+      tags:
+        - archive
+      summary: Delete item from archive
+      description: 'Deletes an item from the archive'
+      operationId: post-api-deleteArchiveItems
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DeleteArchiveItemsRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/importArchive:
+    post:
+      tags:
+        - archive
+      summary: Imports archive
+      description: 'Imports an existing archive.txt file'
+      operationId: post-api-importArchive
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ImportArchiveRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
+  /api/uploadCookies:
+    post:
+      tags:
+        - downloader
+      summary: Upload cookies
+      description: 'Uploads cookies file to be used during downloading'
+      operationId: post-api-uploadCookies
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadCookiesRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -785,7 +905,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/SuccessObject'
+                $ref: '#/components/schemas/RestartDownloadResponse'
       requestBody:
         content:
           application/json:
@@ -841,17 +961,10 @@ paths:
         - Auth query parameter: []
       tags:
         - downloader
-  /api/clearFinishedDownloads:
+  /api/clearDownloads:
     post:
-      tags:
+      summary: Clear multiple downloads
-        - downloader
+      operationId: post-api-clear-downloads
-      summary: Clear finished downloads
-      operationId: post-api-clear-finished-downloads
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
       responses:
         '200':
           description: OK
@@ -859,8 +972,17 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/SuccessObject'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ClearDownloadsRequest'
+        description: ''
+      description: "Clears multiple downloads based on a given filter."
       security:
         - Auth query parameter: []
+      tags:
+        - downloader
   /api/getTask:
     post:
       summary: Get info for one task
@@ -1507,6 +1629,8 @@ components:
       properties:
         success:
           type: boolean
+        error:
+          type: string
     FileType:
       type: string
       enum:
@@ -1559,6 +1683,10 @@ components:
           type: string
           description: Height of the video, if known
           example: '1080'
+        maxHeight:
+          type: string
+          description: Max height that should be used, useful for playlists. selectedHeight will override this.
+          example: '1080'
         maxBitrate:
           type: string
           description: Specify ffmpeg/avconv audio quality
@@ -1567,6 +1695,9 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
+        ignoreArchive:
+          type: boolean
+          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -1591,6 +1722,13 @@ components:
       properties:
         download:
           $ref: '#/components/schemas/Download'
+    RestartDownloadResponse:
+      allOf:
+        - $ref: '#/components/schemas/SuccessObject'
+        - type: object
+          properties:
+            new_download_uid:
+              type: string
     GetAllDownloadsRequest:
       type: object
       properties:
@@ -1607,6 +1745,15 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Download'
+    ClearDownloadsRequest:
+      type: object
+      properties:
+        clear_finished:
+          type: boolean
+        clear_paused:
+          type: boolean
+        clear_errors:
+          type: boolean
     GetTaskRequest:
       type: object
       properties:
@@ -1634,6 +1781,16 @@ components:
       required:
         - task_key
         - new_data
+    UpdateTaskOptionsRequest:
+      type: object
+      properties:
+        task_key:
+          type: string
+        new_options:
+          type: object
+      required:
+        - task_key
+        - new_options
     GetTaskResponse:
       type: object
       properties:
@@ -1690,6 +1847,51 @@ components:
           description: All video playlists
           items:
             $ref: '#/components/schemas/Playlist'
+    GetAllFilesRequest:
+      type: object
+      properties:
+        sort:
+          $ref: '#/components/schemas/Sort'
+        range:
+          type: array
+          items:
+            type: number
+            description: Two elements allowed, start index and end index
+          minItems: 2
+          maxItems: 2
+          default: null
+        text_search:
+          type: string
+          description: Filter files by title
+          default: null
+        file_type_filter:
+          $ref: '#/components/schemas/FileTypeFilter'
+        favorite_filter:
+          type: boolean
+          description: If set to true, only gets favorites
+          default: false
+        sub_id:
+          type: string
+          description: Include if you want to filter by subscription
+          default: null
+    Sort:
+      type: object
+      properties:
+        by:
+          type: string
+          description: Property to sort by
+          default: registered
+        order:
+          type: number
+          description: 1 for ascending, -1 for descending
+          default: -1
+    FileTypeFilter:
+      type: string
+      enum:
+        - audio_only
+        - video_only
+        - both
+      default: both
     GetAllFilesResponse:
       required:
         - files
@@ -1727,6 +1929,18 @@ components:
           type: boolean
         file:
           $ref: '#/components/schemas/DatabaseFile'
+    UpdateFileRequest:
+      required:
+        - uid
+        - change_obj
+      type: object
+      properties:
+        uid:
+          type: string
+          description: Video UID
+        change_obj:
+          type: object
+          description: Object with fields to update as keys and their new values
     SharingToggle:
       required:
         - uid
@@ -1740,7 +1954,6 @@ components:
       required:
         - name
         - url
-        - streamingOnly
       type: object
       properties:
         name:
@@ -1796,16 +2009,11 @@ components:
           description: Number of files removed
     DeleteSubscriptionFileRequest:
       required:
-        - file
+        - file_uid
-        - sub
       type: object
       properties:
-        file:
-          type: string
         file_uid:
           type: string
-        sub:
-          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteForever:
           type: boolean
           description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -1853,7 +2061,6 @@ components:
         - uids
         - playlistName
         - thumbnailURL
-        - type
       type: object
       properties:
         playlistName:
@@ -1862,8 +2069,6 @@ components:
           type: array
           items:
             type: string
-        type:
-          $ref: '#/components/schemas/FileType'
         thumbnailURL:
           type: string
     CreatePlaylistResponse:
@@ -1893,15 +2098,17 @@ components:
       required:
         - playlist
         - success
-        - type
       type: object
       properties:
         playlist:
           $ref: '#/components/schemas/Playlist'
-        type:
-          $ref: '#/components/schemas/FileType'
         success:
           type: boolean
+        file_objs:
+          type: array
+          description: File objects for every uid in the playlist's uids property, in the same order
+          items:
+            $ref: '#/components/schemas/DatabaseFile'
     GetPlaylistsRequest:
       type: object
       properties:
@@ -1926,13 +2133,10 @@ components:
     DeletePlaylistRequest:
       required:
         - playlist_id
-        - type
       type: object
       properties:
         playlist_id:
           type: string
-        type:
-          $ref: '#/components/schemas/FileType'
     DownloadFileRequest:
       type: object
       properties:
@@ -1949,17 +2153,83 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      required:
-        - sub
       type: object
       properties:
-        sub:
+        type:
-          required:
+          $ref: '#/components/schemas/FileType'
-            - archive_dir
+        sub_id:
-          type: object
+          type: string
-          properties:
+    Archive:
-            archive_dir:
+      required:
-              type: string
+        - extractor
+        - id
+        - type
+        - title
+        - timestamp
+        - uid
+      type: object
+      properties:
+        extractor:
+          type: string
+        id:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        title:
+          type: string
+        user_uid:
+          type: string
+        sub_id:
+          type: string
+        timestamp:
+          type: number
+        uid:
+          type: string
+    DeleteArchiveItemsRequest:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    ImportArchiveRequest:
+      type: object
+      required:
+        - archive
+        - type
+      properties:
+        archive:
+          type: string
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesRequest:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/FileType'
+        sub_id:
+          type: string
+    GetArchivesResponse:
+      type: object
+      required:
+        - archives
+      properties:
+        archives:
+          type: array
+          items:
+            $ref: '#/components/schemas/Archive'
+    UploadCookiesRequest:
+      type: object
+      required:
+        - cookies
+      properties:
+        cookies:
+          type: string
+          format: binary
     UpdaterStatus:
       required:
         - details
@@ -1980,8 +2250,6 @@ components:
         tag:
           type: string
     DBInfoResponse:
-      required:
-        - db_info
       type: object
       properties:
         using_local_db:
@@ -2003,6 +2271,8 @@ components:
               $ref: '#/components/schemas/TableInfo'
             download_queue:
               $ref: '#/components/schemas/TableInfo'
+            archives:
+              $ref: '#/components/schemas/TableInfo'
     TransferDBResponse:
       required:
         - success
@@ -2153,7 +2423,6 @@ components:
           type: boolean
         result:
           allOf:
-          - $ref: '#/components/schemas/file'
           - type: object
             properties:
               formats:
@@ -2303,6 +2572,7 @@ components:
         - upload_date
         - uploader
         - url
+        - favorite
       type: object
       properties:
         id:
@@ -2311,6 +2581,9 @@ components:
           type: string
         thumbnailURL:
           type: string
+          description: Backup if thumbnailPath is not defined
+        thumbnailPath:
+          type: string
         isAudio:
           type: boolean
         duration:
@@ -2322,14 +2595,35 @@ components:
           type: string
         size:
           type: number
+          description: In bytes
         path:
           type: string
         upload_date:
           type: string
         uid:
           type: string
+        user_uid:
+          type: string
         sharingEnabled:
           type: boolean
+        category:
+          $ref: '#/components/schemas/Category'
+        view_count:
+          type: number
+        local_view_count:
+          type: number
+        sub_id:
+          type: string
+        registered:
+          type: number
+        height:
+          type: number
+          description: In pixels, only for videos
+        abr:
+          type: number
+          description: In Kbps
+        favorite:
+          type: boolean
     Playlist:
       required:
         - uids
@@ -2359,6 +2653,10 @@ components:
           type: number
         user_uid:
           type: string
+        auto:
+          type: boolean
+        sharingEnabled:
+          type: boolean
     Download:
       required:
         - url
@@ -2403,12 +2701,18 @@ components:
           type: string
           description: Error text, set if download fails.
           nullable: true
+        error_type:
+          type: string
+          description: Error type, may or may not be set in case of an error
+          nullable: true
         user_uid:
           type: string
         sub_id:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key
@@ -2423,6 +2727,8 @@ components:
       properties:
         key:
           type: string
+        title:
+          type: string
         last_ran:
           type: number
         last_confirmed:
@@ -2437,6 +2743,8 @@ components:
           type: string
         schedule:
           type: object
+        options:
+          type: object
     Schedule:
       required:
         - type
@@ -2461,6 +2769,8 @@ components:
               type: number
             timestamp:
               type: number
+            tz:
+              type: string
     DBBackup:
       required:
         - name
@@ -2503,7 +2813,6 @@ components:
         - url
         - type
         - user_uid
-        - streamingOnly
         - isPlaylist
         - videos
       type: object
@@ -2519,8 +2828,6 @@ components:
         user_uid:
           type: string
           nullable: true
-        streamingOnly:
-          type: boolean
         isPlaylist:
           type: boolean
         archive:
@@ -2545,28 +2852,6 @@ components:
           type: string
         passhash:
           type: string
-        files:
-          type: object
-          properties:
-            audio:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-            video:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-        playlists:
-          type: object
-          properties:
-            audio:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-            video:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
         subscriptions:
           type: array
           items:
@@ -2674,6 +2959,44 @@ components:
           type: string
         date:
           type: string
+    Notification:
+      required:
+        - uid
+        - type
+        - text
+        - read
+        - timestamp
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/NotificationType'
+        uid:
+          type: string
+        user_uid:
+          type: string
+        action:
+          type: array
+          items:
+            $ref: '#/components/schemas/NotificationAction'
+        read:
+          type: boolean
+        data:
+          type: object
+        timestamp:
+          type: number
+    NotificationAction:
+      type: string
+      enum:
+      - play
+      - retry_download
+      - view_download_error
+      - view_tasks
+    NotificationType:
+      type: string
+      enum:
+      - download_complete
+      - download_error
+      - task_finished
     BaseChangePermissionsRequest:
       required:
         - permission
@@ -2805,6 +3128,29 @@ components:
                   type: array
                   items:
                     $ref: '#/components/schemas/UserPermission'
+    DeleteNotificationRequest:
+      required:
+        - uid
+      type: object
+      properties:
+        uid:
+          type: string
+    SetNotificationsToReadRequest:
+      required:
+        - uids
+      type: object
+      properties:
+        uids:
+          type: array
+          items:
+            type: string
+    GetNotificationsResponse:
+      type: object
+      properties:
+        notifications:
+          type: array
+          items:
+            $ref: '#/components/schemas/Notification'
   securitySchemes:
     Auth query parameter:
       name: apiKey

README.md

@@ -6,25 +6,15 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
 <hr>
 
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
-Check out the prerequisites, and go to the installation section. Easy as pie!
+Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
@@ -38,13 +28,29 @@ Dark mode:
 
 NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
 
-Debian/Ubuntu:
+Required dependencies:
+
+* Node.js 16
+* Python
+
+Optional dependencies:
+
+* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
+
+
+<details>
+  <summary>Debian/Ubuntu</summary>
 
 ```bash
+curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
 sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
 ```
 
-CentOS 7:
+</details>
+
+<details>
+  <summary>CentOS 7</summary>
 
 ```bash
 sudo yum install epel-release
@@ -52,15 +58,16 @@ sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfu
 sudo yum install centos-release-scl-rh
 sudo yum install rh-nodejs12
 scl enable rh-nodejs12 bash
+curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash -
 sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 ```
 
-Optional dependencies:
+</details>
-
-* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
 
 ### Installing
 
+If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
+
 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.
@@ -91,7 +98,7 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ### Host-specific instructions
 
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
 
 ### Setup
 
@@ -102,8 +109,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for
 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
 
-NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
-
 ### Custom UID/GID
 
 By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
@@ -114,6 +119,12 @@ environment:
     GID: YOUR_GID
 ```
 
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
 ## API
 
 [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+If you would like to see the latest updates, use the `nightly` tag on Docker.
+
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
+
+| Version              | Supported          |
+| -------------------- | ------------------ |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
+
+## Reporting a Vulnerability
+
+Please file an issue in our GitHub's repo, because this app
+isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
+
+We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

angular.json

@@ -30,7 +30,8 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "scripts": [],
             "vendorChunk": true,
@@ -118,7 +119,8 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "scripts": []
           },
@@ -151,7 +153,8 @@
             "tsConfig": "src/tsconfig.spec.json",
             "scripts": [],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "assets": [
               "src/assets",
@@ -179,7 +182,6 @@
       }
     }
   },
-  "defaultProject": "youtube-dl-material",
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",
@@ -188,5 +190,8 @@
     "@schematics/angular:directive": {
       "prefix": "app"
     }
+  },
+  "cli": {
+    "analytics": false
   }
 }

app.json

@@ -2,6 +2,7 @@
     "name": "YoutubeDL-Material",
     "description": "An open-source and self-hosted YouTube downloader based on Google's Material Design specifications.",
     "repository": "https://github.com/Tzahi12345/YoutubeDL-Material",
+    "stack": "container",
     "logo": "https://i.imgur.com/GPzvPiU.png",
     "keywords": ["youtube-dl", "youtubedl-material", "nodejs"]
   }

armhf.Dockerfile

@@ -1,49 +0,0 @@
-FROM alpine:3.12 as frontend
-
-RUN apk add --no-cache \
-  npm \
-  curl
-
-RUN npm install -g @angular/cli
-
-WORKDIR /build
-
-RUN curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
-
-COPY [ "package.json", "package-lock.json", "/build/" ]
-RUN npm install
-
-COPY [ "angular.json", "tsconfig.json", "/build/" ]
-COPY [ "src/", "/build/src/" ]
-RUN ng build --prod
-
-#--------------#
-
-FROM arm32v7/alpine:3.12
-
-COPY --from=frontend /build/qemu-arm-static /usr/bin
-
-ENV UID=1000 \
-  GID=1000 \
-  USER=youtube
-
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
-
-RUN apk add --no-cache \
-  ffmpeg \
-  npm \
-  python2 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
-
-WORKDIR /app
-COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
-RUN npm install && chown -R $UID:$GID ./
-
-COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
-
-EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "node", "app.js" ]

backend/app.js

@@ -2,7 +2,6 @@ const { uuid } = require('uuidv4');
 const fs = require('fs-extra');
 const { promisify } = require('util');
 const auth_api = require('./authentication/auth');
-const winston = require('winston');
 const path = require('path');
 const compression = require('compression');
 const multer  = require('multer');
@@ -18,6 +17,8 @@ const URL = require('url').URL;
 const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
+const Feed = require('feed').Feed;
+const session = require('express-session');
 
 // needed if bin/details somehow gets deleted
 if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -32,6 +33,8 @@ const subscriptions_api = require('./subscriptions');
 const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
+const files_api = require('./files');
 
 var app = express();
 
@@ -68,7 +71,9 @@ db.defaults(
         configWriteFlag: false,
         downloads: {},
         subscriptions: [],
-        files_to_db_migration_complete: false
+        files_to_db_migration_complete: false,
+        tasks_manager_role_migration_complete: false,
+        archives_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -101,7 +106,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -148,22 +152,19 @@ if (fs.existsSync('version.json')) {
 
 // don't overwrite config if it already happened.. NOT
 // let alreadyWritten = db.get('configWriteFlag').value();
-let writeConfigMode = process.env.write_ytdl_config;
 
 // checks if config exists, if not, a config is auto generated
 config_api.configExistsCheck();
 
-if (writeConfigMode) {
+setAndLoadConfig();
-    setAndLoadConfig();
-} else {
-    loadConfig();
-}
 
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
 // use passport
 app.use(auth_api.passport.initialize());
+app.use(session({ secret: uuid(), resave: true, saveUninitialized: true }))
+app.use(auth_api.passport.session());
 
 // actual functions
 
@@ -174,10 +175,10 @@ async function checkMigrations() {
     if (!simplified_db_migration_complete) {
         logger.info('Beginning migration: 4.1->4.2+')
         let success = await simplifyDBFileStructure();
-        success = success && await db_api.addMetadataPropertyToDB('view_count');
+        success = success && await files_api.addMetadataPropertyToDB('view_count');
-        success = success && await db_api.addMetadataPropertyToDB('description');
+        success = success && await files_api.addMetadataPropertyToDB('description');
-        success = success && await db_api.addMetadataPropertyToDB('height');
+        success = success && await files_api.addMetadataPropertyToDB('height');
-        success = success && await db_api.addMetadataPropertyToDB('abr');
+        success = success && await files_api.addMetadataPropertyToDB('abr');
         // sets migration to complete
         db.set('simplified_db_migration_complete', true).write();
         if (success) { logger.info('4.1->4.2+ migration complete!'); }
@@ -188,13 +189,31 @@ async function checkMigrations() {
     if (!new_db_system_migration_complete) {
         logger.info('Beginning migration: 4.2->4.3+')
         let success = await db_api.importJSONToDB(db.value(), users_db.value());
-
+        await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first
         // sets migration to complete
         db.set('new_db_system_migration_complete', true).write();
         if (success) { logger.info('4.2->4.3+ migration complete!'); }
         else { logger.error('Migration failed: 4.2->4.3+'); }
     }
 
+    const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value();
+    if (!tasks_manager_role_migration_complete) {
+        logger.info('Checking if tasks manager role permissions exist for admin user...');
+        const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes');
+        if (success) logger.info('Task manager permissions check complete!');
+        else logger.error('Failed to auto add tasks manager permissions to admin role!');
+        db.set('tasks_manager_role_migration_complete', true).write();
+    }
+
+    const archives_migration_complete = db.get('archives_migration_complete').value();
+    if (!archives_migration_complete) {
+        logger.info('Checking if archives have been migrated...');
+        const imported_archives = await archive_api.importArchives();
+        if (imported_archives) logger.info('Archives migration complete!');
+        else logger.error('Failed to migrate archives!');
+        db.set('archives_migration_complete', true).write();
+    }
+
     return true;
 }
 
@@ -249,14 +268,6 @@ async function startServer() {
     });
 }
 
-async function restartServer(is_update = false) {
-    logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
-
-    // the following line restarts the server through nodemon
-    fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
-    process.exit(1);
-}
-
 async function updateServer(tag) {
     // no tag provided means update to the latest version
     if (!tag) {
@@ -297,7 +308,7 @@ async function updateServer(tag) {
             updating: true,
             'details': 'Update complete! Restarting server...'
         }
-        restartServer(true);
+        utils.restartServer(true);
     }, err => {
         logger.error(err);
         updaterStatus = {
@@ -492,8 +503,9 @@ async function setAndLoadConfig() {
 }
 
 async function setConfigFromEnv() {
-    let config_items = getEnvConfigItems();
+    const config_items = getEnvConfigItems();
-    let success = config_api.setConfigItems(config_items);
+    if (!config_items || config_items.length === 0) return true;
+    const success = config_api.setConfigItems(config_items);
     if (success) {
         logger.info('Config items set using ENV variables.');
         await utils.wait(100);
@@ -508,13 +520,11 @@ async function loadConfig() {
     loadConfigValues();
 
     // connect to DB
-    await db_api.connectToDB();
+    if (!config_api.getConfigItem('ytdl_use_local_db'))
+        await db_api.connectToDB();
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
-    // creates archive path if missing
-    await fs.ensureDir(archivePath);
-
     // check migrations
     await checkMigrations();
 
@@ -560,14 +570,7 @@ function loadConfigValues() {
     url_domain = new URL(url);
 
     let logger_level = config_api.getConfigItem('ytdl_logger_level');
-    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    utils.updateLoggerLevel(logger_level);
-    if (!possible_levels.includes(logger_level)) {
-        logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
-        logger_level = 'info';
-    }
-    logger.level = logger_level;
-    winston.loggers.get('console').level = logger_level;
-    logger.transports[2].level = logger_level;
 }
 
 function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -583,7 +586,11 @@ async function watchSubscriptions() {
 
     if (!subscriptions) return;
 
-    const valid_subscriptions = subscriptions.filter(sub => !sub.paused);
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
 
     let subscriptions_amount = valid_subscriptions.length;
     let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
@@ -676,6 +683,7 @@ async function getUrlInfos(url) {
 
 async function startYoutubeDL() {
     // auto update youtube-dl
+    youtubedl_api.verifyBinaryExistsLinux();
     const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
     if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
 }
@@ -697,7 +705,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -718,7 +726,7 @@ const optionalJwt = async function (req, res, next) {
         const uuid = using_body ? req.body.uuid : req.query.uuid;
         const uid = using_body ? req.body.uid : req.query.uid;
         const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
-        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
+        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
         if (file) {
             req.can_watch = true;
             return next();
@@ -764,13 +772,13 @@ app.get('/api/versionInfo', (req, res) => {
 
 app.post('/api/restartServer', optionalJwt, (req, res) => {
     // delayed by a little bit so that the client gets a response
-    setTimeout(() => {restartServer()}, 100);
+    setTimeout(() => {utils.restartServer()}, 100);
     res.send({success: true});
 });
 
 app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
     const db_info = await db_api.getDBStats();
-    res.send({db_info: db_info});
+    res.send(db_info);
 });
 
 app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -803,18 +811,20 @@ app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
 app.post('/api/downloadFile', optionalJwt, async function(req, res) {
     req.setTimeout(0); // remove timeout in case of long videos
     const url = req.body.url;
-    const type = req.body.type;
+    const type = req.body.type ? req.body.type : 'video';
     const user_uid = req.isAuthenticated() ? req.user.uid : null;
     const options = {
         customArgs: req.body.customArgs,
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings
+        cropFileSettings: req.body.cropFileSettings,
+        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -840,6 +850,7 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
+        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
@@ -918,45 +929,40 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
-    let files = null;
+    const sort = req.body.sort;
-    let playlists = null;
+    const range = req.body.range;
-    let sort = req.body.sort;
+    const text_search = req.body.text_search;
-    let range = req.body.range;
+    const file_type_filter = req.body.file_type_filter;
-    let text_search = req.body.text_search;
+    const favorite_filter = req.body.favorite_filter;
-    let file_type_filter = req.body.file_type_filter;
+    const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const filter_obj = {user_uid: uuid};
+    const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
-    let file_count = await db_api.getRecords('files', filter_obj, true);
-    playlists = await db_api.getRecords('playlists', {user_uid: uuid});
-
-    const categories = await categories_api.getCategoriesAsPlaylists(files);
-    if (categories) {
-        playlists = playlists.concat(categories);
-    }
-
-    files = JSON.parse(JSON.stringify(files));
 
     res.send({
         files: files,
         file_count: file_count,
-        playlists: playlists
     });
 });
 
+app.post('/api/updateFile', optionalJwt, async function (req, res) {
+    const uid = req.body.uid;
+    const change_obj = req.body.change_obj;
+
+    const file = await db_api.updateRecord('files', {uid: uid}, change_obj);
+
+    if (!file) {
+        res.send({
+            success: false,
+            error: 'File could not be found'
+        });
+    } else {
+        res.send({
+            success: true
+        });
+    }
+});
+
 app.post('/api/checkConcurrentStream', async (req, res) => {
     const uid = req.body.uid;
 
@@ -1074,9 +1080,6 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
             await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
         } else if (is_playlist) {
             await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
-        } else if (type === 'subscription') {
-            // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
-            //          time they are requested from the subscription directory.
         } else {
             // error
             success = false;
@@ -1091,7 +1094,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
     });
 });
 
-app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
+app.post('/api/incrementViewCount', async (req, res) => {
     let file_uid = req.body.file_uid;
     let sub_id = req.body.sub_id;
     let uuid = req.body.uuid;
@@ -1100,7 +1103,7 @@ app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
         uuid = req.user.uid;
     }
 
-    const file_obj = await db_api.getVideo(file_uid, uuid, sub_id);
+    const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
 
     const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
     const new_view_count = current_view_count + 1;
@@ -1226,12 +1229,9 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
-    let file = req.body.file;
     let file_uid = req.body.file_uid;
-    let sub = req.body.sub;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
+    let success = await files_api.deleteFile(file_uid, deleteForever);
 
     if (success) {
         res.send({
@@ -1264,7 +1264,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
     subscription = JSON.parse(JSON.stringify(subscription));
 
     // get sub videos
-    if (subscription.name && !subscription.streamingOnly) {
+    if (subscription.name) {
         var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
         subscription['videos'] = parsed_files;
         // loop through files for extra processing
@@ -1274,19 +1274,6 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
         }
 
-        res.send({
-            subscription: subscription,
-            files: parsed_files
-        });
-    } else if (subscription.name && subscription.streamingOnly) {
-        // return list of videos
-        let parsed_files = [];
-        if (subscription.videos) {
-            for (let i = 0; i < subscription.videos.length; i++) {
-                const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
-            }
-        }
         res.send({
             subscription: subscription,
             files: parsed_files
@@ -1331,9 +1318,8 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
 app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
-    let type = req.body.type;
 
-    const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1346,13 +1332,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
     let include_file_metadata = req.body.include_file_metadata;
 
-    const playlist = await db_api.getPlaylist(playlist_id, uuid);
+    const playlist = await files_api.getPlaylist(playlist_id, uuid);
     const file_objs = [];
 
     if (playlist && include_file_metadata) {
         for (let i = 0; i < playlist['uids'].length; i++) {
             const uid = playlist['uids'][i];
-            const file_obj = await db_api.getVideo(uid, uuid);
+            const file_obj = await files_api.getVideo(uid, uuid);
             if (file_obj) file_objs.push(file_obj);
             // TODO: remove file from playlist if could not be found
         }
@@ -1361,7 +1347,6 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     res.send({
         playlist: playlist,
         file_objs: file_objs,
-        type: playlist && playlist.type,
         success: !!playlist
     });
 });
@@ -1372,7 +1357,7 @@ app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
 
     let playlists = await db_api.getRecords('playlists', {user_uid: uuid});
     if (include_categories) {
-        const categories = await categories_api.getCategoriesAsPlaylists(files);
+        const categories = await categories_api.getCategoriesAsPlaylists();
         if (categories) {
             playlists = playlists.concat(categories);
         }
@@ -1391,7 +1376,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
     playlist.uids.push(file_uid);
 
-    let success = await db_api.updatePlaylist(playlist);
+    let success = await files_api.updatePlaylist(playlist);
     res.send({
         success: success
     });
@@ -1399,7 +1384,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
 app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
     let playlist = req.body.playlist;
-    let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
+    let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
     res.send({
         success: success
     });
@@ -1427,10 +1412,9 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
 app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const uid = req.body.uid;
     const blacklistMode = req.body.blacklistMode;
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     let wasDeleted = false;
-    wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
+    wasDeleted = await files_api.deleteFile(uid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1462,7 +1446,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
+        wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1488,10 +1472,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     if (playlist_id) {
         zip_file_generated = true;
         const playlist_files_to_download = [];
-        const playlist = await db_api.getPlaylist(playlist_id, uuid);
+        const playlist = await files_api.getPlaylist(playlist_id, uuid);
         for (let i = 0; i < playlist['uids'].length; i++) {
             const playlist_file_uid = playlist['uids'][i];
-            const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
+            const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
             playlist_files_to_download.push(file_obj);
         }
 
@@ -1505,7 +1489,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
         // generate zip
         file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
     } else {
-        const file_obj = await db_api.getVideo(uid, uuid, sub_id)
+        const file_obj = await files_api.getVideo(uid, uuid, sub_id)
         file_path_to_download = file_obj.path;
     }
     if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
@@ -1523,20 +1507,69 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     });
 });
 
+app.post('/api/getArchives', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const sub_id = req.body.sub_id;
+    const filter_obj = {user_uid: uuid, sub_id: sub_id};
+    const type = req.body.type;
+
+    // we do this for file types because if type is null, that means get files of all types
+    if (type) filter_obj['type'] = type;
+
+    const archives = await db_api.getRecords('archives', filter_obj);
+
+    res.send({
+        archives: archives
+    });
+});
+
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    let sub = req.body.sub;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
-    let archive_dir = sub.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
 
-    let full_archive_path = path.join(archive_dir, 'archive.txt');
+    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
 
-    if (await fs.pathExists(full_archive_path)) {
+    if (archive_text !== null && archive_text !== undefined) {
-        res.sendFile(full_archive_path);
+        res.setHeader('Content-type', "application/octet-stream");
+        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
+        res.send(archive_text);
     } else {
-        res.sendStatus(404);
+        res.sendStatus(400);
     }
 
 });
 
+app.post('/api/importArchive', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archive = req.body.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
+
+    const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
+
+    const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
+
+    res.send({
+        success: !!imported_count,
+        imported_count: imported_count
+    });
+});
+
+app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const archives = req.body.archives;
+
+    let success = true;
+    for (const archive of archives) {
+        success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
+    }
+
+    res.send({
+        success: success
+    });
+});
+
 var upload_multer = multer({ dest: __dirname + '/appdata/' });
 app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
     const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1603,12 +1636,12 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
 
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
-        file_obj = await db_api.getVideo(uid, uuid, sub_id);
+        file_obj = await files_api.getVideo(uid, uuid, sub_id);
         if (file_obj) file_path = file_obj['path'];
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
-        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
+        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1676,9 +1709,15 @@ app.post('/api/download', optionalJwt, async (req, res) => {
     }
 });
 
-app.post('/api/clearFinishedDownloads', optionalJwt, async (req, res) => {
+app.post('/api/clearDownloads', optionalJwt, async (req, res) => {
     const user_uid = req.isAuthenticated() ? req.user.uid : null;
-    const success = db_api.removeAllRecords('download_queue', {finished: true, user_uid: user_uid});
+    const clear_finished = req.body.clear_finished;
+    const clear_paused = req.body.clear_paused;
+    const clear_errors = req.body.clear_errors;
+    let success = true;
+    if (clear_finished) success &= await db_api.removeAllRecords('download_queue', {finished: true,        user_uid: user_uid});
+    if (clear_paused)   success &= await db_api.removeAllRecords('download_queue', {paused:   true,        user_uid: user_uid});
+    if (clear_errors)   success &= await db_api.removeAllRecords('download_queue', {error:    {$ne: null}, user_uid: user_uid});
     res.send({success: success});
 });
 
@@ -1722,8 +1761,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
 
 app.post('/api/restartDownload', optionalJwt, async (req, res) => {
     const download_uid = req.body.download_uid;
-    const success = await downloader_api.restartDownload(download_uid);
+    const new_download = await downloader_api.restartDownload(download_uid);
-    res.send({success: success});
+    res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
 });
 
 app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1800,8 +1839,18 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
+    const task_key = req.body.task_key;
+    const new_options = req.body.new_options;
+  
+    const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
+
+    res.send({success: success});
+});
+
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
+    fs.ensureDirSync(backup_dir);
     const db_backups = [];
 
     const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false);
@@ -1982,6 +2031,93 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+// notifications
+
+app.post('/api/getNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
+
+    res.send({notifications: notifications});
+});
+
+// set notifications to read
+app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
+    const uid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeRecord('notifications', {uid: uid});
+
+    res.send({success: success});
+});
+
+app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
+
+    const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
+
+    res.send({success: success});
+});
+
+// rss feed
+
+app.get('/api/rss', async function (req, res) {
+    if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
+        logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
+        res.sendStatus(403);
+        return;
+    }
+
+    // these are returned
+    const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
+    const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
+    const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
+    const file_type_filter = req.query.file_type_filter;
+    const favorite_filter = req.query.favorite_filter === 'true';
+    const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
+    const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
+
+    const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+
+    const feed = new Feed({
+            title: 'Downloads',
+            description: 'YoutubeDL-Material downloads',
+            id: utils.getBaseURL(),
+            link: utils.getBaseURL(),
+            image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
+            favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
+            generator: 'YoutubeDL-Material'
+    });
+
+    files.forEach(file => {
+        feed.addItem({
+            title: file.title,
+            link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
+            description: file.description,
+            author: [
+                {
+                    name: file.uploader,
+                    link: file.url
+                }
+            ],
+            contributor: [],
+            date: file.timestamp,
+            // https://stackoverflow.com/a/45415677/8088021
+            image: file.thumbnailURL.replace('&', '&amp;')
+        });
+      });
+    res.send(feed.rss2());
+});
+
+// web server
+
 app.use(function(req, res, next) {
     //if the request is not html then move along
     var accept = req.accepts('html', 'json', 'xml');

backend/appdata/default.json

@@ -23,7 +23,12 @@
       "download_only_mode": false,
       "allow_autoplay": true,
       "enable_downloads_manager": true,
-      "allow_playlist_categorization": true
+      "allow_playlist_categorization": true,
+      "force_autoplay": false,
+      "enable_notifications": true,
+      "enable_all_notifications": true,
+      "allowed_notification_types": [],
+      "enable_rss_feed": false
     },
     "API": {
       "use_API_key": false,
@@ -31,10 +36,22 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
-      "generate_NFO_files": false
+      "generate_NFO_files": false,
+      "use_ntfy_API": false,
+      "ntfy_topic_URL": "",
+      "use_gotify_API": false,
+      "gotify_server_URL": "",
+      "gotify_app_token": "",
+      "use_telegram_API": false,
+      "telegram_bot_token": "",
+      "telegram_chat_id": "",
+      "webhook_URL": "",
+      "discord_webhook_URL": "",
+      "slack_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",
@@ -63,7 +80,7 @@
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
-      "default_downloader": "youtube-dl",
+      "default_downloader": "yt-dlp",
       "use_default_downloading_agent": true,
       "custom_downloading_agent": "",
       "multi_user_mode": false,

backend/archive.js

@@ -0,0 +1,91 @@
+const path = require('path');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+
+const db_api = require('./db');
+
+exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
+    const filter = {user_uid: user_uid, sub_id: sub_id};
+    if (type) filter['type'] = type;
+    const archive_items = await db_api.getRecords('archives', filter);
+    const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
+    return archive_item_lines.join('\n');
+}
+
+exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
+    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
+    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
+    return success;
+}
+
+exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return success;
+}
+
+exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
+    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
+    return !!archive_item;
+}
+
+exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
+    let archive_import_count = 0;
+    const lines = archive_text.split('\n');
+    for (let line of lines) {
+        const archive_line_parts = line.trim().split(' ');
+        // should just be the extractor and the video ID
+        if (archive_line_parts.length !== 2) {
+            continue;
+        }
+
+        const extractor = archive_line_parts[0];
+        const id        = archive_line_parts[1];
+        if (!extractor || !id) continue;
+
+        // we can't do a bulk write because we need to avoid duplicate archive items existing in db
+
+        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
+        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
+        archive_import_count++;
+    }
+    return archive_import_count;
+}
+
+exports.importArchives = async () => {
+    const imported_archives = [];
+    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        if (!dir_to_check['archive_path']) continue;
+
+        const files_to_import = [
+            path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
+            path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
+        ]
+
+        for (const file_to_import of files_to_import) {
+            const file_exists = await fs.pathExists(file_to_import);
+            if (!file_exists) continue;
+
+            const archive_text = await fs.readFile(file_to_import, 'utf8');
+            await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
+            imported_archives.push(file_to_import);
+        }
+    }
+    return imported_archives;
+}
+
+const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
+    return {
+        extractor: extractor,
+        id: id,
+        type: type,
+        title: title,
+        user_uid: user_uid ? user_uid : null,
+        sub_id: sub_id ? sub_id : null,
+        timestamp: Date.now() / 1000,
+        uid: uuid()
+    }
+}

backend/authentication/auth.js

@@ -18,13 +18,29 @@ let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function() {
+exports.initialize = function () {
   /*************************
    * Authentication module
    ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
   saltRounds = 10;
 
+  // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
+  if (!(+JWT_EXPIRATION)) {
+    logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
+    JWT_EXPIRATION = 86400;
+  } else {
+    JWT_EXPIRATION = +JWT_EXPIRATION;
+  }
 
   SERVER_SECRET = null;
   if (db_api.users_db.get('jwt_secret').value()) {
@@ -49,6 +65,41 @@ exports.initialize = function() {
   }));
 }
 
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
 exports.passport = require('passport');
 
 exports.passport.serializeUser(function(user, done) {
@@ -127,8 +178,12 @@ exports.registerUser = async function(req, res) {
 
 
 exports.login = async (username, password) => {
+  // even if we're using LDAP, we still want users to be able to login using internal credentials
   const user = await db_api.getRecord('users', {name: username});
-  if (!user) { logger.error(`User ${username} not found`); return false }
+  if (!user) {
+    if (config_api.getConfigItem('ytdl_auth_method') === 'internal') logger.error(`User ${username} not found`);
+    return false;
+  }
   if (user.auth_method && user.auth_method !== 'internal') { return false }
   return await bcrypt.compare(password, user.passhash) ? user : false;
 }
@@ -313,7 +368,6 @@ exports.userHasPermission = async function(user_uid, permission) {
     logger.error('Invalid role ' + role);
     return false;
   }
-  const role_permissions = (await db_api.getRecords('roles'))['permissions'];
 
   const user_has_explicit_permission = user_obj['permissions'].includes(permission);
   const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -328,7 +382,8 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 
   // no overrides, let's check if the role has the permission
-  if (role_permissions.includes(permission)) {
+  const role_has_permission = await exports.roleHasPermissions(role, permission);
+  if (role_has_permission) {
     return true;
   } else {
     logger.verbose(`User ${user_uid} failed to get permission ${permission}`);
@@ -336,6 +391,16 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 }
 
+exports.roleHasPermissions = async function(role, permission) {
+  const role_obj = await db_api.getRecord('roles', {key: role})
+  if (!role) {
+    logger.error(`Role ${role} does not exist!`);
+  }
+  const role_permissions = role_obj['permissions'];
+  if (role_permissions && role_permissions.includes(permission)) return true;
+  else return false;
+}
+
 exports.userPermissions = async function(user_uid) {
   let user_permissions = [];
   const user_obj = await db_api.getRecord('users', ({uid: user_uid}));

backend/categories.js

@@ -55,17 +55,18 @@ async function getCategories() {
     return categories ? categories : null;
 }
 
-async function getCategoriesAsPlaylists(files = null) {
+async function getCategoriesAsPlaylists() {
     const categories_as_playlists = [];
     const available_categories = await getCategories();
-    if (available_categories && files) {
+    if (available_categories) {
         for (let category of available_categories) {
-            const files_that_match = utils.addUIDsToCategory(category, files);
+            const files_that_match = await db_api.getRecords('files', {'category.uid': category['uid']});
             if (files_that_match && files_that_match.length > 0) {
                 category['thumbnailURL'] = files_that_match[0].thumbnailURL;
                 category['thumbnailPath'] = files_that_match[0].thumbnailPath;
                 category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
                 category['id'] = category['uid'];
+                category['auto'] = true;
                 categories_as_playlists.push(category);
             }
         }

backend/config.js

@@ -127,7 +127,7 @@ function setConfigItem(key, value) {
     success = setConfigFile(config_json);
 
     return success;
-};
+}
 
 function setConfigItems(items) {
     let success = false;
@@ -185,7 +185,6 @@ const DEFAULT_CONFIG = {
         "default_file_output": "",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false,
         "include_thumbnail": true,
         "include_metadata": true,
         "max_concurrent_downloads": 5,
@@ -196,20 +195,33 @@ const DEFAULT_CONFIG = {
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_autoplay": true,
+        "force_autoplay": false,
         "enable_downloads_manager": true,
-        "allow_playlist_categorization": true
+        "allow_playlist_categorization": true,
+        "enable_notifications": true,
+        "enable_all_notifications": true,
+        "allowed_notification_types": [],
+        "enable_rss_feed": false,
       },
       "API": {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
-        "use_twitch_API": false,
-        "twitch_API_key": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
-        "generate_NFO_files": false
+        "generate_NFO_files": false,
+        "use_ntfy_API": false,
+        "ntfy_topic_URL": "",
+        "use_gotify_API": false,
+        "gotify_server_URL": "",
+        "gotify_app_token": "",
+        "use_telegram_API": false,
+        "telegram_bot_token": "",
+        "telegram_chat_id": "",
+        "webhook_URL": "",
+        "discord_webhook_URL": "",
+        "slack_webhook_URL": "",
       },
       "Themes": {
         "default_theme": "default",
@@ -238,7 +250,7 @@ const DEFAULT_CONFIG = {
         "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
       },
       "Advanced": {
-        "default_downloader": "youtube-dl",
+        "default_downloader": "yt-dlp",
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",
         "multi_user_mode": false,

backend/consts.js

@@ -30,10 +30,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'
     },
-    'ytdl_safe_download_override': {
-        'key': 'ytdl_safe_download_override',
-        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
-    },
     'ytdl_include_thumbnail': {
         'key': 'ytdl_include_thumbnail',
         'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -68,9 +64,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_autoplay': {
+    'ytdl_force_autoplay': {
-        'key': 'ytdl_allow_autoplay',
+        'key': 'ytdl_force_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+        'path': 'YoutubeDLMaterial.Extra.force_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -80,6 +76,22 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_allow_playlist_categorization',
         'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
+    'ytdl_enable_notifications': {
+        'key': 'ytdl_enable_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_notifications'
+    },
+    'ytdl_enable_all_notifications': {
+        'key': 'ytdl_enable_all_notifications',
+        'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
+    },
+    'ytdl_allowed_notification_types': {
+        'key': 'ytdl_allowed_notification_types',
+        'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
+    },
+    'ytdl_enable_rss_feed': {
+        'key': 'ytdl_enable_rss_feed',
+        'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
+    },
 
     // API
     'ytdl_use_api_key': {
@@ -98,14 +110,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
-    'ytdl_use_twitch_api': {
-        'key': 'ytdl_use_twitch_api',
-        'path': 'YoutubeDLMaterial.API.use_twitch_API'
-    },
-    'ytdl_twitch_api_key': {
-        'key': 'ytdl_twitch_api_key',
-        'path': 'YoutubeDLMaterial.API.twitch_API_key'
-    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -118,6 +122,50 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_generate_nfo_files',
         'path': 'YoutubeDLMaterial.API.generate_NFO_files'
     },
+    'ytdl_use_ntfy_API': {
+        'key': 'ytdl_use_ntfy_API',
+        'path': 'YoutubeDLMaterial.API.use_ntfy_API'
+    },
+    'ytdl_ntfy_topic_url': {
+        'key': 'ytdl_ntfy_topic_url',
+        'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
+    },
+    'ytdl_use_gotify_API': {
+        'key': 'ytdl_use_gotify_API',
+        'path': 'YoutubeDLMaterial.API.use_gotify_API'
+    },
+    'ytdl_gotify_server_url': {
+        'key': 'ytdl_gotify_server_url',
+        'path': 'YoutubeDLMaterial.API.gotify_server_URL'
+    },
+    'ytdl_gotify_app_token': {
+        'key': 'ytdl_gotify_app_token',
+        'path': 'YoutubeDLMaterial.API.gotify_app_token'
+    },
+    'ytdl_use_telegram_API': {
+        'key': 'ytdl_use_telegram_API',
+        'path': 'YoutubeDLMaterial.API.use_telegram_API'
+    },
+    'ytdl_telegram_bot_token': {
+        'key': 'ytdl_telegram_bot_token',
+        'path': 'YoutubeDLMaterial.API.telegram_bot_token'
+    },
+    'ytdl_telegram_chat_id': {
+        'key': 'ytdl_telegram_chat_id',
+        'path': 'YoutubeDLMaterial.API.telegram_chat_id'
+    },
+    'ytdl_webhook_url': {
+        'key': 'ytdl_webhook_url',
+        'path': 'YoutubeDLMaterial.API.webhook_URL'
+    },
+    'ytdl_discord_webhook_url': {
+        'key': 'ytdl_discord_webhook_url',
+        'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
+    },
+    'ytdl_slack_webhook_url': {
+        'key': 'ytdl_slack_webhook_url',
+        'path': 'YoutubeDLMaterial.API.slack_webhook_URL'
+    },
 
 
     // Themes
@@ -217,9 +265,91 @@ exports.AVAILABLE_PERMISSIONS = [
     'subscriptions',
     'sharing',
     'advanced_download',
-    'downloads_manager'
+    'downloads_manager',
+    'tasks_manager'
 ];
 
 exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
 
-exports.CURRENT_VERSION = 'v4.2';
+// args that have a value after it (e.g. -o <output> or -f <format>)
+const YTDL_ARGS_WITH_VALUES = [
+    '--default-search',
+    '--config-location',
+    '--proxy',
+    '--socket-timeout',
+    '--source-address',
+    '--geo-verification-proxy',
+    '--geo-bypass-country',
+    '--geo-bypass-ip-block',
+    '--playlist-start',
+    '--playlist-end',
+    '--playlist-items',
+    '--match-title',
+    '--reject-title',
+    '--max-downloads',
+    '--min-filesize',
+    '--max-filesize',
+    '--date',
+    '--datebefore',
+    '--dateafter',
+    '--min-views',
+    '--max-views',
+    '--match-filter',
+    '--age-limit',
+    '--download-archive',
+    '-r',
+    '--limit-rate',
+    '-R',
+    '--retries',
+    '--fragment-retries',
+    '--buffer-size',
+    '--http-chunk-size',
+    '--external-downloader',
+    '--external-downloader-args',
+    '-a',
+    '--batch-file',
+    '-o',
+    '--output',
+    '--output-na-placeholder',
+    '--autonumber-start',
+    '--load-info-json',
+    '--cookies',
+    '--cache-dir',
+    '--encoding',
+    '--user-agent',
+    '--referer',
+    '--add-header',
+    '--sleep-interval',
+    '--max-sleep-interval',
+    '-f',
+    '--format',
+    '--merge-output-format',
+    '--sub-format',
+    '--sub-lang',
+    '-u',
+    '--username',
+    '-p',
+    '--password',
+    '-2',
+    '--twofactor',
+    '--video-password',
+    '--ap-mso',
+    '--ap-username',
+    '--ap-password',
+    '--audio-format',
+    '--audio-quality',
+    '--recode-video',
+    '--postprocessor-args',
+    '--metadata-from-title',
+    '--fixup',
+    '--ffmpeg-location',
+    '--exec',
+    '--convert-subs'
+];
+
+// we're using a Set here for performance
+exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
+
+exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
+
+exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -1,10 +1,11 @@
-var fs = require('fs-extra')
+const fs = require('fs-extra')
-var path = require('path')
+const path = require('path')
 const { MongoClient } = require("mongodb");
 const { uuid } = require('uuidv4');
+const _ = require('lodash');
 
 const config_api = require('./config');
-var utils = require('./utils')
+const utils = require('./utils')
 const logger = require('./logger');
 
 const low = require('lowdb')
@@ -58,6 +59,13 @@ const tables = {
         name: 'tasks',
         primary_key: 'key'
     },
+    notifications: {
+        name: 'notifications',
+        primary_key: 'uid'
+    },
+    archives: {
+        name: 'archives'
+    },
     test: {
         name: 'test'
     }
@@ -85,7 +93,6 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
-    if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;
 
@@ -149,6 +156,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
                 await database.collection(table).createIndex(text_search);
             }
         });
+        using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
         return true;
     } catch(err) {
         logger.error(err);
@@ -159,82 +167,9 @@ exports._connectToDB = async (custom_connection_string = null) => {
     }
 }
 
-exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
+exports.setVideoProperty = async (file_uid, assignment_obj) => {
-    if (!file_object) file_object = generateFileObject(file_path, type);
+    // TODO: check if video exists, throw error if not
-    if (!file_object) {
+    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
-        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
-        return false;
-    }
-
-    utils.fixVideoMetadataPerms(file_path, type);
-
-    // add thumbnail path
-    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
-
-    // if category exists, only include essential info
-    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
-
-    // modify duration
-    if (cropFileSettings) {
-        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
-    }
-
-    if (user_uid) file_object['user_uid'] = user_uid;
-    if (sub_id) file_object['sub_id'] = sub_id;
-
-    const file_obj = await registerFileDBManual(file_object);
-
-    // remove metadata JSON if needed
-    if (!config_api.getConfigItem('ytdl_include_metadata')) {
-        utils.deleteJSONFile(file_path, type)
-    }
-
-    return file_obj;
-}
-
-async function registerFileDBManual(file_object) {
-    // add additional info
-    file_object['uid'] = uuid();
-    file_object['registered'] = Date.now();
-    path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
-
-    exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
-
-    return file_object;
-}
-
-function generateFileObject(file_path, type) {
-    var jsonobj = utils.getJSON(file_path, type);
-    if (!jsonobj) {
-        return null;
-    } else if (!jsonobj['_filename']) {
-        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
-        return null;
-    }
-    const ext = (type === 'audio') ? '.mp3' : '.mp4'
-    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
-    // console.
-    var stats = fs.statSync(true_file_path);
-
-    const file_id = utils.removeFileExtension(path.basename(file_path));
-    var title = jsonobj.title;
-    var url = jsonobj.webpage_url;
-    var uploader = jsonobj.uploader;
-    var upload_date = utils.formatDateString(jsonobj.upload_date);
-
-    var size = stats.size;
-
-    var thumbnail = jsonobj.thumbnail;
-    var duration = jsonobj.duration;
-    var isaudio = type === 'audio';
-    var description = jsonobj.description;
-    var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
-    return file_obj;
-}
-
-function getAppendedBasePathSub(sub, base_path) {
-    return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
 exports.getFileDirectoriesAndDBs = async () => {
@@ -253,13 +188,16 @@ exports.getFileDirectoriesAndDBs = async () => {
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'audio'),
                 user_uid: user.uid,
-                type: 'audio'
+                type: 'audio',
+                archive_path: utils.getArchiveFolder('audio', user.uid)
             });
 
             // add user's video dir to check list
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'video'),
-                type: 'video'
+                user_uid: user.uid,
+                type: 'video',
+                archive_path: utils.getArchiveFolder('video', user.uid)
             });
         }
     } else {
@@ -269,13 +207,15 @@ exports.getFileDirectoriesAndDBs = async () => {
         // add audio dir to check list
         dirs_to_check.push({
             basePath: audioFolderPath,
-            type: 'audio'
+            type: 'audio',
+            archive_path: utils.getArchiveFolder('audio')
         });
 
         // add video dir to check list
         dirs_to_check.push({
             basePath: videoFolderPath,
-            type: 'video'
+            type: 'video',
+            archive_path: utils.getArchiveFolder('video')
         });
     }
 
@@ -296,261 +236,14 @@ exports.getFileDirectoriesAndDBs = async () => {
                                       : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
             user_uid: subscription_to_check.user_uid,
             type: subscription_to_check.type,
-            sub_id: subscription_to_check['id']
+            sub_id: subscription_to_check['id'],
+            archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
         });
     }
 
     return dirs_to_check;
 }
 
-exports.importUnregisteredFiles = async () => {
-    const imported_files = [];
-    const dirs_to_check = await exports.getFileDirectoriesAndDBs();
-
-    // run through check list and check each file to see if it's missing from the db
-    for (let i = 0; i < dirs_to_check.length; i++) {
-        const dir_to_check = dirs_to_check[i];
-        // recursively get all files in dir's path
-        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
-
-        for (let j = 0; j < files.length; j++) {
-            const file = files[j];
-
-            // check if file exists in db, if not add it
-            const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
-            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
-            if (!file_is_registered) {
-                // add additional info
-                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
-                if (file_obj) {
-                    imported_files.push(file_obj['uid']);
-                    logger.verbose(`Added discovered file to the database: ${file.id}`);
-                } else {
-                    logger.error(`Failed to import ${file['path']} automatically.`);
-                }
-            }
-        }
-    }
-    return imported_files;
-}
-
-exports.addMetadataPropertyToDB = async (property_key) => {
-    try {
-        const dirs_to_check = await exports.getFileDirectoriesAndDBs();
-        const update_obj = {};
-        for (let i = 0; i < dirs_to_check.length; i++) {
-            const dir_to_check = dirs_to_check[i];
-
-            // recursively get all files in dir's path
-            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
-            for (let j = 0; j < files.length; j++) {
-                const file = files[j];
-                if (file[property_key]) {
-                    update_obj[file.uid] = {[property_key]: file[property_key]};
-                }
-            }
-        }
-
-        return await exports.bulkUpdateRecords('files', 'uid', update_obj);
-    } catch(err) {
-        logger.error(err);
-        return false;
-    }
-}
-
-exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
-    const first_video = await exports.getVideo(uids[0]);
-    const thumbnailToUse = first_video['thumbnailURL'];
-    
-    let new_playlist = {
-        name: playlist_name,
-        uids: uids,
-        id: uuid(),
-        thumbnailURL: thumbnailToUse,
-        type: type,
-        registered: Date.now(),
-        randomize_order: false
-    };
-
-    new_playlist.user_uid = user_uid ? user_uid : undefined;
-
-    await exports.insertRecordIntoTable('playlists', new_playlist);
-    
-    const duration = await exports.calculatePlaylistDuration(new_playlist);
-    await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
-
-    return new_playlist;
-}
-
-exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
-    let playlist = await exports.getRecord('playlists', {id: playlist_id});
-
-    if (!playlist) {
-        playlist = await exports.getRecord('categories', {uid: playlist_id});
-        if (playlist) {
-            // category found
-            const files = await exports.getFiles(user_uid);
-            utils.addUIDsToCategory(playlist, files);
-        }
-    }
-
-    // converts playlists to new UID-based schema
-    if (playlist && playlist['fileNames'] && !playlist['uids']) {
-        playlist['uids'] = [];
-        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
-        for (let i = 0; i < playlist['fileNames'].length; i++) {
-            const fileName = playlist['fileNames'][i];
-            const uid = await exports.getVideoUIDByID(fileName, user_uid);
-            if (uid) playlist['uids'].push(uid);
-            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
-        }
-        exports.updatePlaylist(playlist, user_uid);
-    }
-
-    // prevent unauthorized users from accessing the file info
-    if (require_sharing && !playlist['sharingEnabled']) return null;
-
-    return playlist;
-}
-
-exports.updatePlaylist = async (playlist) => {
-    let playlistID = playlist.id;
-
-    const duration = await exports.calculatePlaylistDuration(playlist);
-    playlist.duration = duration;
-
-    return await exports.updateRecord('playlists', {id: playlistID}, playlist);
-}
-
-exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
-    let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
-
-    if (!success) {
-        success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
-    }
-
-    if (!success) {
-        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
-    }
-
-    return success;
-}
-
-exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
-    if (!playlist_file_objs) {
-        playlist_file_objs = [];
-        for (let i = 0; i < playlist['uids'].length; i++) {
-            const uid = playlist['uids'][i];
-            const file_obj = await exports.getVideo(uid);
-            if (file_obj) playlist_file_objs.push(file_obj);
-        }
-    }
-
-    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
-}
-
-exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid, uuid);
-    const type = file_obj.isAudio ? 'audio' : 'video';
-    const folderPath = path.dirname(file_obj.path);
-    const ext = type === 'audio' ? 'mp3' : 'mp4';
-    const name = file_obj.id;
-    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
-
-    var jsonPath = `${file_obj.path}.info.json`;
-    var altJSONPath = `${filePathNoExtension}.info.json`;
-    var thumbnailPath = `${filePathNoExtension}.webp`;
-    var altThumbnailPath = `${filePathNoExtension}.jpg`;
-
-    jsonPath = path.join(__dirname, jsonPath);
-    altJSONPath = path.join(__dirname, altJSONPath);
-
-    let jsonExists = await fs.pathExists(jsonPath);
-    let thumbnailExists = await fs.pathExists(thumbnailPath);
-
-    if (!jsonExists) {
-        if (await fs.pathExists(altJSONPath)) {
-            jsonExists = true;
-            jsonPath = altJSONPath;
-        }
-    }
-
-    if (!thumbnailExists) {
-        if (await fs.pathExists(altThumbnailPath)) {
-            thumbnailExists = true;
-            thumbnailPath = altThumbnailPath;
-        }
-    }
-
-    let fileExists = await fs.pathExists(file_obj.path);
-
-    if (config_api.descriptors[uid]) {
-        try {
-            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
-                config_api.descriptors[uid][i].destroy();
-            }
-        } catch(e) {
-
-        }
-    }
-
-    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-    if (useYoutubeDLArchive) {
-        const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
-
-        // get ID from JSON
-
-        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let id = null;
-        if (jsonobj) id = jsonobj.id;
-
-        // use subscriptions API to remove video from the archive file, and write it to the blacklist
-        if (await fs.pathExists(archive_path)) {
-            const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
-            if (blacklistMode && line) await writeToBlacklist(type, line);
-        } else {
-            logger.info('Could not find archive file for audio files. Creating...');
-            await fs.close(await fs.open(archive_path, 'w'));
-        }
-    }
-
-    if (jsonExists) await fs.unlink(jsonPath);
-    if (thumbnailExists) await fs.unlink(thumbnailPath);
-
-    await exports.removeRecord('files', {uid: uid});
-
-    if (fileExists) {
-        await fs.unlink(file_obj.path);
-        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
-            return false;
-        } else {
-            return true;
-        }
-    } else {
-        // TODO: tell user that the file didn't exist
-        return true;
-    }
-}
-
-// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
-exports.getVideoUIDByID = async (file_id, uuid = null) => {
-    const file_obj = await exports.getRecord('files', {id: file_id});
-    return file_obj ? file_obj['uid'] : null;
-}
-
-exports.getVideo = async (file_uid) => {
-    return await exports.getRecord('files', {uid: file_uid});
-}
-
-exports.getFiles = async (uuid = null) => {
-    return await exports.getRecords('files', {user_uid: uuid});
-}
-
-exports.setVideoProperty = async (file_uid, assignment_obj) => {
-    // TODO: check if video exists, throw error if not
-    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
-}
-
 // Basic DB functions
 
 // Create
@@ -558,7 +251,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
 exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
     // local db override
     if (using_local_db) {
-        if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
+        if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
         local_db.get(table).push(doc).write();
         return true;
     }
@@ -629,7 +322,7 @@ exports.bulkInsertRecordsIntoTable = async (table, docs) => {
 exports.getRecord = async (table, filter_obj) => {
     // local db override
     if (using_local_db) {
-        return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
+        return exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
     }
 
     return await database.collection(table).findOne(filter_obj);
@@ -638,7 +331,7 @@ exports.getRecord = async (table, filter_obj) => {
 exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
     // local db override
     if (using_local_db) {
-        let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        let cursor = filter_obj ? exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
         if (sort) {
             cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
         }
@@ -661,10 +354,16 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 
 // Update
 
-exports.updateRecord = async (table, filter_obj, update_obj) => {
+exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
+        if (nested_mode) {
+            // if object is nested we need to handle it differently
+            update_obj = utils.convertFlatObjectToNestedObject(update_obj);
+            exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
+            return true;
+        }
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
 
@@ -677,7 +376,14 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
+            const props_to_update = Object.keys(update_obj);
+            for (let i = 0; i < props_to_update.length; i++) {
+                const prop_to_update = props_to_update[i];
+                const prop_value = update_obj[prop_to_update];
+                record[prop_to_update] = prop_value;
+            }
+        }).write();
         return true;
     }
 
@@ -685,7 +391,19 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
     return !!(output['result']['ok']);
 }
 
-exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
+exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
+    // local db override
+    if (using_local_db) {
+        const props_to_remove = Object.keys(remove_obj);
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
+        return true;
+    }
+
+    const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
+    return !!(output['result']['ok']);
+}
+
+exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
     // local db override
     if (using_local_db) {
         local_db.get(table).each((record) => {
@@ -722,7 +440,7 @@ exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
 exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
         return true;
     }
 
@@ -733,7 +451,7 @@ exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
 exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
         return true;
     }
 
@@ -746,7 +464,7 @@ exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
 exports.removeRecord = async (table, filter_obj) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
         return true;
     }
 
@@ -757,7 +475,7 @@ exports.removeRecord = async (table, filter_obj) => {
 // exports.removeRecordsByUIDBulk = async (table, uids) => {
 //     // local db override
 //     if (using_local_db) {
-//         applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+//         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
 //         return true;
 //     }
 
@@ -821,7 +539,7 @@ exports.removeAllRecords = async (table = null, filter_obj = null) => {
     if (using_local_db) {
         for (let i = 0; i < tables_to_remove.length; i++) {
             const table_to_remove = tables_to_remove[i];
-            if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+            if (filter_obj) exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
             else local_db.assign({[table_to_remove]: []}).write();
             logger.debug(`Successfully removed records from ${table_to_remove}`);
         }
@@ -933,6 +651,7 @@ exports.importJSONToDB = async (db_json, users_json) => {
 const createFilesRecords = (files, subscriptions) => {
     for (let i = 0; i < subscriptions.length; i++) {
         const subscription = subscriptions[i];
+        if (!subscription['videos']) continue;
         subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
         files = files.concat(subscriptions[i]['videos']);
     }
@@ -993,7 +712,7 @@ exports.backupDB = async () => {
     const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
     const path_to_backups = path.join(backup_dir, backup_file_name);
 
-    logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
+    logger.info(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
 
     const table_to_records = {};
     for (let i = 0; i < tables_list.length; i++) {
@@ -1040,10 +759,11 @@ exports.transferDB = async (local_to_remote) => {
         table_to_records[table] = await exports.getRecords(table);
     }
 
+    logger.info('Backup up DB...');
+    await exports.backupDB(); // should backup always
+
     using_local_db = !local_to_remote;
     if (local_to_remote) {
-        logger.debug('Backup up DB...');
-        await exports.backupDB();
         const db_connected = await exports.connectToDB(5, true);
         if (!db_connected) {
             logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1075,8 +795,13 @@ exports.transferDB = async (local_to_remote) => {
     This function is necessary to emulate mongodb's ability to search for null or missing values.
         A filter of null or undefined for a property will find docs that have that property missing, or have it
         null or undefined. We want that same functionality for the local DB as well
+
+        error:    {$ne: null}
+          ^            ^
+          |            |
+      filter_prop  filter_prop_value
 */
-const applyFilterLocalDB = (db_path, filter_obj, operation) => {
+exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     const filter_props = Object.keys(filter_obj);
     const return_val = db_path[operation](record => {
         if (!filter_props) return true;
@@ -1085,14 +810,28 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
             const filter_prop = filter_props[i];
             const filter_prop_value = filter_obj[filter_prop];
             if (filter_prop_value === undefined || filter_prop_value === null) {
-                filtered &= record[filter_prop] === undefined || record[filter_prop] === null
+                filtered &= record[filter_prop] === undefined || record[filter_prop] === null;
             } else {
                 if (typeof filter_prop_value === 'object') {
-                    if (filter_prop_value['$regex']) {
+                    if ('$regex' in filter_prop_value) {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
+                    } else if ('$ne' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
+                    } else if ('$lt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
+                    } else if ('$gt' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
+                    } else if ('$lte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
+                    } else if ('$gte' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
                     }
                 } else {
-                    filtered &= record[filter_prop] === filter_prop_value;
+                    // handle case of nested property check
+                    if (filter_prop.includes('.'))
+                        filtered &= utils.searchObjectByString(record, filter_prop) === filter_prop_value;
+                    else
+                        filtered &= record[filter_prop] === filter_prop_value;
                 }
             }
         }
@@ -1101,14 +840,7 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
     return return_val;
 }
 
-// archive helper functions
+// should only be used for tests
-
+exports.setLocalDBMode = (mode) => {
-async function writeToBlacklist(type, line) {
+    using_local_db = mode;
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
+}
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}

backend/downloader.js

@@ -1,9 +1,7 @@
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const path = require('path');
-const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
-const glob = require('glob')
 const Mutex = require('async-mutex').Mutex;
 
 const youtubedl = require('youtube-dl');
@@ -15,12 +13,13 @@ const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
+const files_api = require('./files');
+const notifications_api = require('./notifications');
+const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
 if (db_api.database_initialized) {
     setupDownloads();
 } else {
@@ -29,7 +28,26 @@ if (db_api.database_initialized) {
     });
 }
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+/*
+
+This file handles all the downloading functionality.
+
+To download a file, we go through 4 steps. Here they are with their respective index & function:
+
+0: Create the download
+ - createDownload()
+1: Get info for the download (we need this step for categories and archive functionality)
+ - collectInfo()
+2: Download the file
+ - downloadQueuedFile()
+3: Complete
+ - N/A
+
+We use checkDownloads() to move downloads through the steps and call their respective functions.
+
+*/
+
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -38,6 +56,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             user_uid: user_uid,
             sub_id: sub_id,
             sub_name: sub_name,
+            prefetched_info: prefetched_info,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -86,10 +105,10 @@ exports.resumeDownload = async (download_uid) => {
 exports.restartDownload = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     await exports.clearDownload(download_uid);
-    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
     
     should_check_downloads = true;
-    return success;
+    return new_download;
 }
 
 exports.cancelDownload = async (download_uid) => {
@@ -108,8 +127,10 @@ exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download_uid, error_message) {
+async function handleDownloadError(download, error_message, error_type = null) {
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+    if (!download || !download['uid']) return;
+    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
+    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
 async function setupDownloads() {
@@ -155,6 +176,13 @@ async function checkDownloads() {
         if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
 
         if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            if (waiting_download['sub_id']) {
+                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
+                if (sub_missing) {
+                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    continue;
+                }
+            }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
@@ -187,13 +215,28 @@ async function collectInfo(download_uid) {
     let args = await exports.generateArgs(url, type, options, download['user_uid']);
 
     // get video info prior to download
-    let info = await getVideoInfoByURL(url, args, download_uid);
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
     if (!info) {
         // info failed, error presumably already recorded
         return;
     }
 
+    // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
+        if (exists_in_archive) {
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            logger.warn(error);
+            if (download_uid) {
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error, 'exists_in_archive');
+                return;
+            }
+        }
+    }
+
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -204,9 +247,11 @@ async function collectInfo(download_uid) {
         options.customOutput = category['custom_output'];
         options.noRelativePath = true;
         args = await exports.generateArgs(url, type, options, download['user_uid']);
-        info = await getVideoInfoByURL(url, args, download_uid);
+        info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
+    const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
+
     // setup info required to calculate download progress
 
     const expected_file_size = utils.getExpectedFileSize(info);
@@ -227,7 +272,9 @@ async function collectInfo(download_uid) {
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    category: stripped_category,
+                                                                    prefetched_info: null
                                                                 });
 }
 
@@ -240,6 +287,7 @@ async function downloadQueuedFile(download_uid) {
     return new Promise(async resolve => {
         const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
         await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
 
         const url = download['url'];
@@ -247,9 +295,11 @@ async function downloadQueuedFile(download_uid) {
         const options = download['options'];
         const args = download['args'];
         const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
         if (options.customFileFolderPath) {
             fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
         }
         fs.ensureDirSync(fileFolderPath);
 
@@ -266,14 +316,14 @@ async function downloadQueuedFile(download_uid) {
             clearInterval(download_checker);
             if (err) {
                 logger.error(err.stderr);
-                await handleDownloadError(download_uid, err.stderr);
+                await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_output');
                     logger.warn(error_message);
                     resolve(false);
                     return;
@@ -282,7 +332,10 @@ async function downloadQueuedFile(download_uid) {
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
-                        output_json = JSON.parse(output[i]);
+                        // we have to do this because sometimes there will be leading characters before the actual json
+                        const start_idx = output[i].indexOf('{"');
+                        const clean_output = output[i].slice(start_idx, output[i].length);
+                        output_json = JSON.parse(clean_output);
                     } catch(e) {
                         output_json = null;
                     }
@@ -299,7 +352,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -333,31 +386,27 @@ async function downloadQueuedFile(download_uid) {
                     }
 
                     // registers file in DB
-                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+                    const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+
+                    await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
+
+                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
 
                     file_objs.push(file_obj);
                 }
 
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
-                }
-
                 let container = null;
 
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                    container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
                 } else {
                     const error_message = 'Downloaded file failed to result in metadata object.';
                     logger.error(error_message);
-                    await handleDownloadError(download_uid, error_message);
+                    await handleDownloadError(download, error_message, 'no_metadata');
                 }
 
                 const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -371,15 +420,27 @@ async function downloadQueuedFile(download_uid) {
 // helper functions
 
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+
+    if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
+        logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
+    }
+
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
 
     const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
     const globalArgs = config_api.getConfigItem('ytdl_custom_args');
     const useCookies = config_api.getConfigItem('ytdl_use_cookies');
     const is_audio = type === 'audio';
 
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
 
     if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
 
@@ -389,6 +450,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 
     // video-specific args
     const selectedHeight = options.selectedHeight;
+    const maxHeight = options.maxHeight;
+    const heightParam = selectedHeight || maxHeight;
 
     // audio-specific args
     const maxBitrate = options.maxBitrate;
@@ -402,8 +465,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     if (!is_audio && !is_youtube) {
         // tiktok videos fail when using the default format
         qualityPath = null;
-    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
-        qualityPath = ['-f', 'bestvideo+bestaudio']
     }
 
     if (customArgs) {
@@ -411,8 +472,9 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     } else {
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
-        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
+        } else if (heightParam && heightParam !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+            const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
+            qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -449,28 +511,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -486,7 +526,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
         }
 
         if (options.additionalArgs && options.additionalArgs !== '') {
-            downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
+            downloadConfig = utils.injectArgs(downloadConfig, options.additionalArgs.split(',,'));
         }
 
         const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
@@ -494,9 +534,11 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.push('-r', rate_limit);
         }
         
-        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
         if (default_downloader === 'yt-dlp') {
-            downloadConfig.push('--no-clean-infojson');
+            downloadConfig = utils.filterArgs(downloadConfig, ['--print-json']);
+
+            // in yt-dlp -j --no-simulate is preferable
+            downloadConfig.push('--no-clean-info-json', '-j', '--no-simulate');
         }
 
     }
@@ -504,14 +546,15 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     // filter out incompatible args
     downloadConfig = filterArgs(downloadConfig, is_audio);
 
-    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
     return downloadConfig;
 }
 
-async function getVideoInfoByURL(url, args = [], download_uid = null) {
+exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
-        const new_args = [...args];
+        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...temp_args];
 
         const archiveArgIndex = new_args.indexOf('--download-archive');
         if (archiveArgIndex !== -1) {
@@ -543,7 +586,8 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
                     const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
                     logger.error(error);
                     if (download_uid) {
-                        await handleDownloadError(download_uid, error);
+                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download, error, 'parse_failed');
                     }
                     resolve(null);
                 }
@@ -552,7 +596,8 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
                 if (err.stderr) error_message += `\n\n${err.stderr}`;
                 logger.error(error_message);
                 if (download_uid) {
-                    await handleDownloadError(download_uid, error_message);
+                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
                 }
                 resolve(null);
             }
@@ -563,8 +608,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
 function filterArgs(args, isAudio) {
     const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
     const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
-    return args.filter(x => !args_to_remove.includes(x));
 }
 
 async function checkDownloadPercent(download_uid) {
@@ -583,20 +627,26 @@ async function checkDownloadPercent(download_uid) {
     if (!resulting_file_size) return;
 
     let sum_size = 0;
-    glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
+    for (let i = 0; i < files_to_check_for_progress.length; i++) {
-        files.forEach(async file => {
+        const file_to_check_for_progress = files_to_check_for_progress[i];
-            try {
+        const dir = path.dirname(file_to_check_for_progress);
-                const file_stats = fs.statSync(file);
+        if (!fs.existsSync(dir)) continue;
-                if (file_stats && file_stats.size) {
+        fs.readdir(dir, async (err, files) => {
-                    sum_size += file_stats.size;
+            for (let j = 0; j < files.length; j++) {
-                }
+                const file = files[j];
-            } catch (e) {
+                if (!file.includes(path.basename(file_to_check_for_progress))) continue;
-
+                try {
+                    const file_stats = fs.statSync(path.join(dir, file));
+                    if (file_stats && file_stats.size) {
+                        sum_size += file_stats.size;
+                    }
+                } catch (e) {}
             }
+            
+            const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+            await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
         });
-        const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+    }
-        await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
-    });
 }
 
 exports.generateNFOFile = (info, output_path) => {
@@ -613,13 +663,3 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join(archivePath);
-    }
-}

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="pm2-runtime pm2.config.js"
+CMD="npm start"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then
@@ -10,8 +10,8 @@ fi
 
 # chown current working directory to current user
 if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
-  exec su-exec "$UID:$GID" "$0" "$@"
+  exec gosu "$UID:$GID" "$0" "$@"
 fi
 
 exec "$@"

backend/files.js

@@ -0,0 +1,350 @@
+const fs = require('fs-extra')
+const path = require('path')
+const { uuid } = require('uuidv4');
+
+const config_api = require('./config');
+const db_api = require('./db');
+const archive_api = require('./archive');
+const utils = require('./utils')
+const logger = require('./logger');
+
+exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
+    if (!file_object) file_object = generateFileObject(file_path, type);
+    if (!file_object) {
+        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
+        return false;
+    }
+
+    utils.fixVideoMetadataPerms(file_path, type);
+
+    // add thumbnail path
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
+
+    // if category exists, only include essential info
+    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
+
+    // modify duration
+    if (cropFileSettings) {
+        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
+    }
+
+    if (user_uid) file_object['user_uid'] = user_uid;
+    if (sub_id) file_object['sub_id'] = sub_id;
+
+    const file_obj = await registerFileDBManual(file_object);
+
+    // remove metadata JSON if needed
+    if (!config_api.getConfigItem('ytdl_include_metadata')) {
+        utils.deleteJSONFile(file_path, type)
+    }
+
+    return file_obj;
+}
+
+async function registerFileDBManual(file_object) {
+    // add additional info
+    file_object['uid'] = uuid();
+    file_object['registered'] = Date.now();
+    const path_object = path.parse(file_object['path']);
+    file_object['path'] = path.format(path_object);
+
+    await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+
+    return file_object;
+}
+
+function generateFileObject(file_path, type) {
+    const jsonobj = utils.getJSON(file_path, type);
+    if (!jsonobj) {
+        return null;
+    } else if (!jsonobj['_filename']) {
+        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
+        return null;
+    }
+    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
+    // console.
+    const stats = fs.statSync(true_file_path);
+
+    const file_id = utils.removeFileExtension(path.basename(file_path));
+    const title = jsonobj.title;
+    const url = jsonobj.webpage_url;
+    const uploader = jsonobj.uploader;
+    const upload_date = utils.formatDateString(jsonobj.upload_date);
+
+    const size = stats.size;
+
+    const thumbnail = jsonobj.thumbnail;
+    const duration = jsonobj.duration;
+    const isaudio = type === 'audio';
+    const description = jsonobj.description;
+    const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
+    return file_obj;
+}
+
+exports.importUnregisteredFiles = async () => {
+    const imported_files = [];
+    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        // recursively get all files in dir's path
+        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
+
+        for (let j = 0; j < files.length; j++) {
+            const file = files[j];
+
+            // check if file exists in db, if not add it
+            const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
+            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
+            if (!file_is_registered) {
+                // add additional info
+                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
+                if (file_obj) {
+                    imported_files.push(file_obj['uid']);
+                    logger.verbose(`Added discovered file to the database: ${file.id}`);
+                } else {
+                    logger.error(`Failed to import ${file['path']} automatically.`);
+                }
+            }
+        }
+    }
+    return imported_files;
+}
+
+exports.addMetadataPropertyToDB = async (property_key) => {
+    try {
+        const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
+        const update_obj = {};
+        for (let i = 0; i < dirs_to_check.length; i++) {
+            const dir_to_check = dirs_to_check[i];
+
+            // recursively get all files in dir's path
+            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (file[property_key]) {
+                    update_obj[file.uid] = {[property_key]: file[property_key]};
+                }
+            }
+        }
+
+        return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj);
+    } catch(err) {
+        logger.error(err);
+        return false;
+    }
+}
+
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
+    const first_video = await exports.getVideo(uids[0]);
+    const thumbnailToUse = first_video['thumbnailURL'];
+    
+    let new_playlist = {
+        name: playlist_name,
+        uids: uids,
+        id: uuid(),
+        thumbnailURL: thumbnailToUse,
+        registered: Date.now(),
+        randomize_order: false
+    };
+
+    new_playlist.user_uid = user_uid ? user_uid : undefined;
+
+    await db_api.insertRecordIntoTable('playlists', new_playlist);
+    
+    const duration = await exports.calculatePlaylistDuration(new_playlist);
+    await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
+
+    return new_playlist;
+}
+
+exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
+    let playlist = await db_api.getRecord('playlists', {id: playlist_id});
+
+    if (!playlist) {
+        playlist = await db_api.getRecord('categories', {uid: playlist_id});
+        if (playlist) {
+            const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
+            playlist['uids'] = uids;
+            playlist['auto'] = true;
+        }
+    }
+
+    // converts playlists to new UID-based schema
+    if (playlist && playlist['fileNames'] && !playlist['uids']) {
+        playlist['uids'] = [];
+        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
+        for (let i = 0; i < playlist['fileNames'].length; i++) {
+            const fileName = playlist['fileNames'][i];
+            const uid = await exports.getVideoUIDByID(fileName, user_uid);
+            if (uid) playlist['uids'].push(uid);
+            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
+        }
+        exports.updatePlaylist(playlist, user_uid);
+    }
+
+    // prevent unauthorized users from accessing the file info
+    if (require_sharing && !playlist['sharingEnabled']) return null;
+
+    return playlist;
+}
+
+exports.updatePlaylist = async (playlist) => {
+    let playlistID = playlist.id;
+
+    const duration = await exports.calculatePlaylistDuration(playlist);
+    playlist.duration = duration;
+
+    return await db_api.updateRecord('playlists', {id: playlistID}, playlist);
+}
+
+exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
+    let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj);
+
+    if (!success) {
+        success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj);
+    }
+
+    if (!success) {
+        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
+    }
+
+    return success;
+}
+
+exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
+    if (!playlist_file_objs) {
+        playlist_file_objs = [];
+        for (let i = 0; i < playlist['uids'].length; i++) {
+            const uid = playlist['uids'][i];
+            const file_obj = await exports.getVideo(uid);
+            if (file_obj) playlist_file_objs.push(file_obj);
+        }
+    }
+
+    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
+}
+
+exports.deleteFile = async (uid, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid);
+    const type = file_obj.isAudio ? 'audio' : 'video';
+    const folderPath = path.dirname(file_obj.path);
+    const name = file_obj.id;
+    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
+
+    var jsonPath = `${file_obj.path}.info.json`;
+    var altJSONPath = `${filePathNoExtension}.info.json`;
+    var thumbnailPath = `${filePathNoExtension}.webp`;
+    var altThumbnailPath = `${filePathNoExtension}.jpg`;
+
+    jsonPath = path.join(__dirname, jsonPath);
+    altJSONPath = path.join(__dirname, altJSONPath);
+
+    let jsonExists = await fs.pathExists(jsonPath);
+    let thumbnailExists = await fs.pathExists(thumbnailPath);
+
+    if (!jsonExists) {
+        if (await fs.pathExists(altJSONPath)) {
+            jsonExists = true;
+            jsonPath = altJSONPath;
+        }
+    }
+
+    if (!thumbnailExists) {
+        if (await fs.pathExists(altThumbnailPath)) {
+            thumbnailExists = true;
+            thumbnailPath = altThumbnailPath;
+        }
+    }
+
+    let fileExists = await fs.pathExists(file_obj.path);
+
+    if (config_api.descriptors[uid]) {
+        try {
+            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
+                config_api.descriptors[uid][i].destroy();
+            }
+        } catch(e) {
+
+        }
+    }
+
+    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive || file_obj.sub_id) {
+        // get id/extractor from JSON
+
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedID = null;
+        let retrievedExtractor = null;
+        if (info_json) {
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
+
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        if (!blacklistMode) {
+            await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id)
+        } else {
+            const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
+            if (!exists_in_archive) {
+                await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id);
+            }
+        }
+    }
+
+    if (jsonExists) await fs.unlink(jsonPath);
+    if (thumbnailExists) await fs.unlink(thumbnailPath);
+
+    await db_api.removeRecord('files', {uid: uid});
+
+    if (fileExists) {
+        await fs.unlink(file_obj.path);
+        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
+            return false;
+        } else {
+            return true;
+        }
+    } else {
+        // TODO: tell user that the file didn't exist
+        return true;
+    }
+}
+
+// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
+exports.getVideoUIDByID = async (file_id, uuid = null) => {
+    const file_obj = await db_api.getRecord('files', {id: file_id});
+    return file_obj ? file_obj['uid'] : null;
+}
+
+exports.getVideo = async (file_uid) => {
+    return await db_api.getRecord('files', {uid: file_uid});
+}
+
+exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (favorite_filter) {
+        filter_obj['favorite'] = true;
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search)));
+    const file_count = await db_api.getRecords('files', filter_obj, true);
+
+    return {files, file_count};
+}

backend/fix-scripts/001-fix_download_permissions.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-03
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines / Docker env defaults
+PATH_SUBS=/app/subscriptions
+PATH_AUDIO=/app/audio
+PATH_VIDS=/app/video
+
+clear -x
+echo "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+echo "Welcome to the INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M."
+echo "This script will set YTDL-M's download paths' owner to ${USER} (${UID}:${GID})"
+echo "and permissions to the default of 644."
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+echo "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBS ] && i=$((i+1)) && echo "✔ (${i}/3) Found Subscriptions directory at ${PATH_SUBS}"
+[ -d $PATH_AUDIO ] && i=$((i+1)) && echo "✔ (${i}/3) Found Audio directory at ${PATH_AUDIO}"
+[ -d $PATH_VIDS ] && i=$((i+1)) && echo "✔ (${i}/3) Found Video directory at ${PATH_VIDS}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    echo "\nCouldn't find any download path to fix permissions for! \nPlease edit this script to configure!"
+    exit 2;;
+  3)
+    echo "\nFound all download paths to fix permissions for. \nProceed? (Y/N)";;
+  *)
+    echo "\nOnly found ${i} out of 3 download paths! Something about this script's config must be wrong. \nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    echo "\n  Running jobs now... (this may take a while)\n"
+    [ -d $PATH_SUBS ] && chown "$UID:$GID" -R $PATH_SUBS && echo "✔ Set owner of ${PATH_SUBS} to ${USER}."
+    [ -d $PATH_SUBS ] && chmod 644 -R $PATH_SUBS && echo "✔ Set permissions of ${PATH_SUBS} to 644."
+    [ -d $PATH_AUDIO ] && chown "$UID:$GID" -R $PATH_AUDIO &&  echo "✔ Set owner of ${PATH_AUDIO} to ${USER}."
+    [ -d $PATH_AUDIO ] && chmod 644 -R $PATH_AUDIO && echo "✔ Set permissions of ${PATH_AUDIO} to 644."
+    [ -d $PATH_VIDS ] && chown "$UID:$GID" -R $PATH_VIDS &&  echo "✔ Set owner of ${PATH_VIDS} to ${USER}."
+    [ -d $PATH_VIDS ] && chmod 644 -R $PATH_VIDS && echo "✔ Set permissions of ${PATH_VIDS} to 644."
+    echo "\n✔ Done."
+    echo "\n  If you noticed file access errors those MAY be due to currently running downloads."
+    echo "  Feel free to re-run this script, however download parts should have correct file permissions anyhow. :)"
+    exit
+else
+    echo "\nOkay, bye."
+fi

backend/fix-scripts/002-fix_dupes_per_archive_file.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-09
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines (NO TRAILING SLASHES) / Docker env defaults
+PATH_SUBSARCHIVE=/app/subscriptions/archives
+PATH_ONEOFFARCHIVE=/app/appdata/archives
+
+# Backup paths (substitute with your personal preference if you like)
+PATH_SUBSARCHIVEBKP=$PATH_SUBSARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+PATH_ONEOFFARCHIVEBKP=$PATH_ONEOFFARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+
+
+# Define Colors for TUI
+yellow=$(tput setaf 3)
+normal=$(tput sgr0)
+
+tput civis                                                          # hide the cursor
+
+clear -x
+printf "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "Welcome to the INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M."
+printf "\nThis script will cycle through the archive files in the folders mentioned"
+printf "\nbelow and remove within each archive the dupe entries. (compact them)"
+printf "\nDuring some older builds of YTDL-M the archives could receive dupe"
+printf "\nentries and blow up in size, sometimes causing conflicts with download management."
+printf '\n%*s' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBSARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found Subscriptions archive directory at ${PATH_SUBSARCHIVE}"
+[ -d $PATH_ONEOFFARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found one-off archive directory at ${PATH_ONEOFFARCHIVE}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    printf "\n\n  Couldn't find any archive location path! \n\nPlease edit this script to configure!"
+    tput cnorm
+    exit 2;;
+  2)
+    printf "\n\n  Found all archive locations. \n\nProceed? (Y/N)";;
+  *)
+    printf "\n\n  Only found ${i} out of 2 archive locations! Something about this script's config must be wrong. \n\nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg          # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    printf "\n\nRunning jobs now... (this may take a while)\n"
+
+    printf "\nBacking up directories...\n"
+
+    chars="⣾⣽⣻⢿⡿⣟⣯⣷"
+    cp -R $PATH_SUBSARCHIVE $PATH_SUBSARCHIVEBKP &
+    PID=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.15
+    done
+    [ -d $PATH_SUBSARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_SUBSARCHIVE} to ${PATH_SUBSARCHIVEBKP} ($(du -sh $PATH_SUBSARCHIVEBKP | cut -f1))\n"
+
+    cp -R $PATH_ONEOFFARCHIVE $PATH_ONEOFFARCHIVEBKP &
+    PID2=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID2 ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.1
+    done
+    [ -d $PATH_ONEOFFARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_ONEOFFARCHIVE} to ${PATH_ONEOFFARCHIVEBKP} ($(du -sh $PATH_ONEOFFARCHIVEBKP | cut -f1))\n"
+
+
+    printf "\nCompacting files...\n"
+
+    tmpfile=$(mktemp) &&
+
+    [ -d $PATH_SUBSARCHIVE ] &&
+    find $PATH_SUBSARCHIVE -name '*.txt' -print0 | while read -d $'\0' file # Set delimiter to null because we want to catch all possible filenames (WE CANNOT CHANGE IFS HERE) - https://stackoverflow.com/a/15931055
+    do
+        cp "$file" "$tmpfile"
+        { awk '!x[$0]++' "$tmpfile" > "$file"; } &                         # https://unix.stackexchange.com/questions/159695/how-does-awk-a0-work
+        PID3=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID3 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [[ "$AFTER" -ne "$BEFORE" ]]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action needed for file: ${file}\n"
+        fi
+    done
+
+    [ -d $PATH_ONEOFFARCHIVE ] &&
+    find $PATH_ONEOFFARCHIVE -name '*.txt' -print0 | while read -d $'\0' file
+    do
+        cp "$file" "$tmpfile" &
+        awk '!x[$0]++' "$tmpfile" > "$file" &
+        PID4=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID4 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [ "$BEFORE" -ne "$AFTER" ]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action ran for file: ${file}\n"
+        fi
+    done
+    tput cnorm                                                     # show the cursor
+    rm "$tmpfile"
+
+    printf "\n\n✔ Done."
+    printf "\nℹ Please keep in mind that you may still want to"
+    printf "\n  run corruption checks against your archives!\n\n"
+    exit
+else
+    tput cnorm
+    printf "\nOkay, bye.\n\n"
+    exit
+fi

backend/notifications.js

@@ -0,0 +1,249 @@
+const db_api = require('./db');
+const config_api = require('./config');
+const logger = require('./logger');
+const utils = require('./utils');
+const consts = require('./consts');
+
+const { uuid } = require('uuidv4');
+
+const fetch = require('node-fetch');
+const { gotify } = require("gotify");
+const TelegramBot = require('node-telegram-bot-api');
+const REST = require('@discordjs/rest').REST;
+const API = require('@discordjs/core').API;
+const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
+
+const NOTIFICATION_TYPE_TO_TITLE = {
+    task_finished: 'Task finished',
+    download_complete: 'Download complete',
+    download_error: 'Download error'
+}
+
+const NOTIFICATION_TYPE_TO_BODY = {
+    task_finished: (notification) => notification['data']['task_title'],
+    download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
+    download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
+}
+
+const NOTIFICATION_TYPE_TO_URL = {
+    task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
+    download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
+    download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
+}
+
+const NOTIFICATION_TYPE_TO_THUMBNAIL = {
+    task_finished: () => null,
+    download_complete: (notification) => notification['data']['file_thumbnail'],
+    download_error: () => null
+}
+
+exports.sendNotification = async (notification) => {
+    // info necessary if we are using 3rd party APIs
+    const type = notification['type'];
+
+    const data = {
+        title: NOTIFICATION_TYPE_TO_TITLE[type],
+        body: NOTIFICATION_TYPE_TO_BODY[type](notification),
+        type: type,
+        url: NOTIFICATION_TYPE_TO_URL[type](notification),
+        thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
+    }
+
+    if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
+        sendNtfyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
+        sendGotifyNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
+        sendTelegramNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_webhook_url')) {
+        sendGenericNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
+        sendDiscordNotification(data);
+    }
+    if (config_api.getConfigItem('ytdl_slack_webhook_url')) {
+        sendSlackNotification(data);
+    }
+
+    await db_api.insertRecordIntoTable('notifications', notification);
+    return notification;
+}
+
+exports.sendTaskNotification = async (task_obj, confirmed) => {
+    if (!notificationEnabled('task_finished')) return;
+    // workaround for tasks which are user_uid agnostic
+    const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
+    await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
+    const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
+    const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadNotification = async (file, user_uid) => {
+    if (!notificationEnabled('download_complete')) return;
+    const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
+    const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
+    if (!notificationEnabled('download_error')) return;
+    const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
+    const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
+    return await exports.sendNotification(notification);
+}
+
+exports.createNotification = (type, actions, data, user_uid) => {
+    const notification = {
+        type: type,
+        actions: actions,
+        data: data,
+        user_uid: user_uid,
+        uid: uuid(),
+        read: false,
+        timestamp: Date.now()/1000
+    }
+    return notification;
+}
+
+function notificationEnabled(type) {
+    return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
+}
+
+function sendNtfyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to ntfy');
+    fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
+        method: 'POST',
+        body: body,
+        headers: {
+            'Title': title,
+            'Tags': type,
+            'Click': url,
+            'Attach': thumbnail
+        }
+    });
+}
+
+async function sendGotifyNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to gotify');
+    await gotify({
+        server: config_api.getConfigItem('ytdl_gotify_server_url'),
+        app: config_api.getConfigItem('ytdl_gotify_app_token'),
+        title: title,
+        message: body,
+        tag: type,
+        priority: 5, // Keeping default from docs, may want to change this,
+        extras: {
+            "client::notification": {
+                click: { url: url },
+                bigImageUrl: thumbnail
+            }
+        }
+      });
+}
+
+async function sendTelegramNotification({body, title, type, url, thumbnail}) {
+    logger.verbose('Sending notification to Telegram');
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    const bot = new TelegramBot(bot_token);
+    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+}
+
+async function sendDiscordNotification({body, title, type, url, thumbnail}) {
+    const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
+    const url_split = discord_webhook_url.split('webhooks/');
+    const [webhook_id, webhook_token] = url_split[1].split('/');
+    const rest = new REST({ version: '10' });
+    const api = new API(rest);
+    const embed = new EmbedBuilder()
+        .setTitle(title)
+        .setColor(0x00FFFF)
+        .setURL(url)
+        .setDescription(`ID: ${type}`);
+    if (thumbnail) embed.setThumbnail(thumbnail);
+    if (type === 'download_error') embed.setColor(0xFC2003);
+
+    const result = await api.webhooks.execute(webhook_id, webhook_token, {
+        content: body,
+        username: 'YoutubeDL-Material',
+        avatar_url: consts.ICON_URL,
+        embeds: [embed],
+    });
+    return result;
+}
+
+function sendSlackNotification({body, title, type, url, thumbnail}) {
+    const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
+    logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
+    const data = {
+        blocks: [
+            {
+                type: "section",
+                text: {
+                    type: "mrkdwn",
+                    text: `*${title}*`
+                }
+            },
+            {
+                type: "section",
+                text: {
+                    type: "plain_text",
+                    text: body
+                }
+            }
+        ]
+    }
+
+    // add thumbnail if exists
+    if (thumbnail) {
+        data['blocks'].push({
+            type: "image",
+            image_url: thumbnail,
+            alt_text: "notification_thumbnail"
+        });
+    }
+
+    data['blocks'].push(
+        {
+            type: "section",
+            text: {
+                type: "mrkdwn",
+                text: `<${url}|${url}>`
+            }
+        },
+        {
+            type: "context",
+            elements: [
+                {
+                    type: "mrkdwn",
+                    text: `*ID:* ${type}`
+                }
+            ]
+        }
+    );
+
+    fetch(slack_webhook_url, {
+        method: 'POST',
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify(data),
+    });
+}
+
+function sendGenericNotification(data) {
+    const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
+    logger.verbose(`Sending generic notification to ${webhook_url}`);
+    fetch(webhook_url, {
+        method: 'POST',
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify(data),
+    });
+}

backend/package.json

@@ -5,20 +5,9 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon app.js",
+    "start": "pm2-runtime --raw pm2.config.js",
     "debug": "set YTDL_MODE=debug && node app.js"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "*.js",
-      "appdata/*",
-      "public/*"
-    ],
-    "watch": [
-      "restart_update.json",
-      "restart_general.json"
-    ]
-  },
   "repository": {
     "type": "git",
     "url": ""
@@ -28,34 +17,42 @@
   "bugs": {
     "url": ""
   },
+  "engines": {
+    "node": "^16",
+    "npm": "6.14.4"
+  },
   "homepage": "",
   "dependencies": {
+    "@discordjs/builders": "^1.6.1",
+    "@discordjs/core": "^0.5.2",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
-    "async-mutex": "^0.3.1",
+    "async-mutex": "^0.4.0",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
+    "express-session": "^1.17.3",
+    "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
-    "glob": "^7.1.6",
+    "gotify": "^1.1.0",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.0",
+    "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
-    "merge-files": "^0.1.2",
     "mocha": "^9.2.2",
-    "moment": "^2.29.2",
+    "moment": "^2.29.4",
     "mongodb": "^3.6.9",
-    "multer": "^1.4.2",
+    "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
-    "node-id3": "^0.1.14",
+    "node-id3": "^0.2.6",
     "node-schedule": "^2.1.0",
-    "nodemon": "^2.0.7",
+    "node-telegram-bot-api": "^0.61.0",
-    "passport": "^0.5.2",
+    "passport": "^0.6.0",
     "passport-http": "^0.3.0",
-    "passport-jwt": "^4.0.0",
+    "passport-jwt": "^4.0.1",
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
@@ -64,7 +61,7 @@
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.0.6",
+    "uuidv4": "^6.2.13",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"

backend/pm2.config.js

@@ -6,4 +6,4 @@ module.exports = {
       out_file: "/dev/null",
       error_file: "/dev/null"
     }]
-}
+}

backend/subscriptions.js

@@ -3,6 +3,7 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
+const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
 
@@ -91,7 +92,10 @@ async function getSubscriptionInfo(sub) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
+                            let sub_name = sub.name;
+                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
+                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
                         }
                     }
 
@@ -138,27 +142,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
 
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     if (deleteMode && (await fs.pathExists(appendedBasePath))) {
-        if (sub.archive && (await fs.pathExists(sub.archive))) {
-            const archive_file_path = path.join(sub.archive, 'archive.txt');
-            // deletes archive if it exists
-            if (await fs.pathExists(archive_file_path)) {
-                await fs.unlink(archive_file_path);
-            }
-            await fs.rmdir(sub.archive);
-        }
         await fs.remove(appendedBasePath);
     }
+
+    await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
 async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    if (typeof sub === 'string') {
+        // TODO: fix bad workaround where sub is a sub_id
+        sub = await db_api.getRecord('subscriptions', {sub_id: sub});
+    }
     // TODO: combine this with deletefile
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
+    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -177,7 +179,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
+        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = info_json['id'];
+        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -195,12 +199,14 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (!deleteForever && useArchive && sub.archive && retrievedID) {
+            if (deleteForever) {
-                const archive_path = path.join(sub.archive, 'archive.txt')
+                // ensure video is in the archives
-                // if archive exists, remove line with video ID
+                const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
-                if (await fs.pathExists(archive_path)) {
+                if (!exists_in_archive) {
-                    utils.removeIDFromArchive(archive_path, retrievedID);
+                    await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
                 }
+            } else {
+                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
         }
@@ -231,77 +237,41 @@ async function getVideosForSub(sub, user_uid = null) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable')) {
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        // TODO: reimplement
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
-
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
-                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        resolve(files_to_download);
-                        // for (let i = 0; i < outputs.length; i++) {
-                        //     const output = JSON.parse(outputs[i]);
-                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                        //         // we found a video that errored! add it to the archive to prevent future errors
-                        //         if (sub.archive) {
-                        //             archive_dir = sub.archive;
-                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                        //             fs.appendFileSync(archive_path, output['id']);
-                        //         }
-                        //     }
-                        // }
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
                     }
+                } else {
+                    logger.error('Subscription check failed!');
                 }
                 resolve(false);
             } else if (output) {
-                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
-                    logger.verbose('No additional videos to download for ' + sub.name);
-                    resolve(true);
-                    return;
-                }
-                const output_jsons = [];
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                        output_jsons.push(output_json);
-                    } catch(e) {
-                        output_json = null;
-                    }
-                    if (!output_json) {
-                        continue;
-                    }
-                }
-
-                const files_to_download = await getFilesToDownload(sub, output_jsons);
-                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
-
-                for (let j = 0; j < files_to_download.length; j++) {
-                    const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
-                }
-
                 resolve(files_to_download);
-
+           }
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                resolve(true);
-            }
         });
     }, err => {
         logger.error(err);
@@ -309,6 +279,43 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+async function handleOutputJSON(output, sub, user_uid) {
+    if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+        await setFreshUploads(sub, user_uid);
+        checkVideosForFreshUploads(sub, user_uid);
+    }
+
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+        logger.verbose('No additional videos to download for ' + sub.name);
+        return [];
+    }
+
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
+    const files_to_download = await getFilesToDownload(sub, output_jsons);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+    for (let j = 0; j < files_to_download.length; j++) {
+        const file_to_download = files_to_download[j];
+        file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
+    }
+
+    return files_to_download;
+}
+
 function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
@@ -319,10 +326,10 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
 
     const base_download_options = {
-        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        maxHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
         customFileFolderPath: getAppendedBasePath(sub, basePath),
         customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        customArchivePath: path.join(basePath, 'archives', sub.name),
         additionalArgs: sub.custom_args
     }
 
@@ -337,8 +344,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -364,6 +369,13 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
+    // skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+    logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
+    const archive_path = path.join(appendedBasePath, 'archive.txt');
+    await fs.writeFile(archive_path, archive_text);
+    downloadConfig.push('--download-archive', archive_path);
+
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
@@ -374,22 +386,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
-    let archive_dir = null;
-    let archive_path = null;
-
-    if (useArchive && !redownload) {
-        if (sub.archive) {
-            archive_dir = sub.archive;
-            archive_path = path.join(archive_dir, 'archive.txt')
-        }
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
-    // if streaming only mode, just get the list of videos
-    if (sub.streamingOnly) {
-        downloadConfig = ['-f', 'best', '--dump-json'];
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -414,9 +410,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
     if (default_downloader === 'yt-dlp') {
-        downloadConfig.push('--no-clean-infojson');
+        downloadConfig.push('--no-clean-info-json');
     }
 
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
     return downloadConfig;
 }
 
@@ -430,7 +428,11 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+                continue;
             }
+            const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+            if (exists_in_archive) continue;
+
             files_to_download.push(output_json);
         }
     }
@@ -449,7 +451,12 @@ async function getAllSubscriptions() {
 }
 
 async function getSubscription(subID) {
-    return await db_api.getRecord('subscriptions', {id: subID});
+    // stringify and parse because we may override the 'downloading' property
+    const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
+    // now with the download_queue, we may need to override 'downloading'
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
+    if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
+    return sub;
 }
 
 async function getSubscriptionByName(subName, user_uid = null) {
@@ -463,7 +470,7 @@ async function updateSubscription(sub) {
 
 async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
-        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
+        await updateSubscriptionProperty(sub, assignment_obj);
     });
 }
 
@@ -473,22 +480,25 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-async function setFreshUploads(sub, user_uid) {
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    if (!sub_files) return;
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
+    sub_files.forEach(async file => {
-        if (current_date === video['upload_date'].replace(/-/g, '')) {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const video_uid = video['uid'];
+            const file_uid = file['uid'];
-            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
+    sub_files.forEach(async file => {
-        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(video, sub, user_uid)
+            await checkVideoIfBetterExists(file, sub, user_uid)
         }
     });
 }
@@ -510,13 +520,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
 // helper functions

backend/tasks.js

@@ -1,5 +1,8 @@
 const db_api = require('./db');
+const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
+const archive_api = require('./archive');
+const files_api = require('./files');
 
 const fs = require('fs-extra');
 const logger = require('./logger');
@@ -18,7 +21,7 @@ const TASKS = {
         job: null
     },
     missing_db_records: {
-        run: db_api.importUnregisteredFiles,
+        run: files_api.importUnregisteredFiles,
         title: 'Import missing DB records',
         job: null
     },
@@ -33,6 +36,28 @@ const TASKS = {
         confirm: youtubedl_api.updateYoutubeDL,
         title: 'Update youtube-dl',
         job: null
+    },
+    delete_old_files: {
+        run: checkForAutoDeleteFiles,
+        confirm: autoDeleteFiles,
+        title: 'Delete old files',
+        job: null
+    },
+    import_legacy_archives: {
+        run: archive_api.importArchives,
+        title: 'Import legacy archives',
+        job: null
+    }
+}
+
+const defaultOptions = {
+    all: {
+        auto_confirm: false
+    },
+    delete_old_files: {
+        blacklist_files: false,
+        blacklist_subscription_files: false,
+        threshold_days: ''
     }
 }
 
@@ -42,10 +67,10 @@ function scheduleJob(task_key, schedule) {
     if (schedule['type'] === 'timestamp') {
         converted_schedule = new Date(schedule['data']['timestamp']);
     } else if (schedule['type'] === 'recurring') {
-        const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
+        const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
-        const hour = schedule['data']['hour']           ? schedule['data']['hour']      : null;
+        const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
-        const minute = schedule['data']['minute']       ? schedule['data']['minute']    : null;
+        const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
         return null;
@@ -57,7 +82,7 @@ function scheduleJob(task_key, schedule) {
             logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
             return;
         }
-        
+
         // remove schedule if it's a one-time task
         if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
         // we're just "running" the task, any confirmation should be user-initiated
@@ -77,9 +102,10 @@ exports.setupTasks = async () => {
     const tasks_keys = Object.keys(TASKS);
     for (let i = 0; i < tasks_keys.length; i++) {
         const task_key = tasks_keys[i];
+        const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
         const task_in_db = await db_api.getRecord('tasks', {key: task_key});
         if (!task_in_db) {
-            // insert task metadata into table if missing
+            // insert task metadata into table if missing, eventually move title to UI
             await db_api.insertRecordIntoTable('tasks', {
                 key: task_key,
                 title: TASKS[task_key]['title'],
@@ -90,9 +116,19 @@ exports.setupTasks = async () => {
                 data: null,
                 error: null,
                 schedule: null,
-                options: {}
+                options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
             });
         } else {
+            // verify all options exist in task
+            for (const key of Object.keys(mergedDefaultOptions)) {
+                const option_key = `options.${key}`;
+                // Remove any potential mangled option keys (#861)
+                await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
+                if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
+                    await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
+                }
+            }
+
             // reset task if necessary
             await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
 
@@ -123,15 +159,23 @@ exports.executeTask = async (task_key) => {
 
 exports.executeRun = async (task_key) => {
     logger.verbose(`Running task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     // don't set running to true when backup up DB as it will be stick "running" if restored
     if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
     const data = await TASKS[task_key].run();
     await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
     logger.verbose(`Finished running task ${task_key}`);
+    const task_obj = await db_api.getRecord('tasks', {key: task_key});
+    await notifications_api.sendTaskNotification(task_obj, false);
+
+    if (task_obj['options'] && task_obj['options']['auto_confirm']) {
+        exports.executeConfirm(task_key);
+    }
 }
 
 exports.executeConfirm = async (task_key) => {
     logger.verbose(`Confirming task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     if (!TASKS[task_key]['confirm']) {
         return null;
     }
@@ -141,6 +185,7 @@ exports.executeConfirm = async (task_key) => {
     await TASKS[task_key].confirm(data);
     await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
     logger.verbose(`Finished confirming task ${task_key}`);
+    await notifications_api.sendTaskNotification(task_obj, false);
 }
 
 exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -148,6 +193,7 @@ exports.updateTaskSchedule = async (task_key, schedule) => {
     await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
     if (TASKS[task_key]['job']) {
         TASKS[task_key]['job'].cancel();
+        TASKS[task_key]['job'] = null;
     }
     if (schedule) {
         TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
@@ -192,4 +238,31 @@ async function removeDuplicates(data) {
     }
 }
 
+// auto delete files
+
+async function checkForAutoDeleteFiles() {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
+        const error_message = 'Failed to do delete check because no limit was set!';
+        logger.error(error_message);
+        await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
+        return null;
+    }
+    const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
+    const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
+    const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
+    return {files_to_remove: files_to_remove};
+}
+
+async function autoDeleteFiles(data) {
+    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
+    if (data['files_to_remove']) {
+        logger.info(`Removing ${data['files_to_remove'].length} old files!`);
+        for (let i = 0; i < data['files_to_remove'].length; i++) {
+            const file_to_remove = data['files_to_remove'][i];
+            await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
+        }
+    }
+}
+
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -1,8 +1,9 @@
-var assert = require('assert');
+/* eslint-disable no-undef */
+const assert = require('assert');
 const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
 
-process.chdir('./backend')
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -37,20 +38,43 @@ var auth_api = require('../authentication/auth');
 var db_api = require('../db');
 const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
+const archive_api = require('../archive');
+const categories_api = require('../categories');
+const files_api = require('../files');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
 
 db_api.initialize(db, users_db);
 
+const sample_video_json = {
+    id: "Sample Video",
+    title: "Sample Video",
+    thumbnailURL: "https://sampleurl.jpg",
+    isAudio: false,
+    duration: 177.413,
+    url: "sampleurl.com",
+    uploader: "Sample Uploader",
+    size: 2838445,
+    path: "users\\admin\\video\\Sample Video.mp4",
+    upload_date: "2017-07-28",
+    description: null,
+    view_count: 230,
+    abr: 128,
+    thumbnailPath: null,
+    user_uid: "admin",
+    uid: "1ada04ab-2773-4dd4-bbdd-3e2d40761c50",
+    registered: 1628469039377
+}
 
 describe('Database', async function() {
     describe('Import', async function() {
-        it('Migrate', async function() {
+        // it('Migrate', async function() {
-            await db_api.connectToDB();
+        //     await db_api.connectToDB();
-            await db_api.removeAllRecords();
+        //     await db_api.removeAllRecords();
-            const success = await db_api.importJSONToDB(db.value(), users_db.value());
+        //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
-            assert(success);
+        //     assert(success);
-        });
+        // });
 
         it('Transfer to remote', async function() {
             await db_api.removeAllRecords('test');
@@ -83,158 +107,233 @@ describe('Database', async function() {
         });
     });
 
-    describe('Export', function() {
-
-    });
-
-
     describe('Basic functions', async function() {
-        beforeEach(async function() {
+        
-            await db_api.connectToDB();
+        // test both local_db and remote_db
-            await db_api.removeAllRecords('test');
+        const local_db_modes = [false, true];
-        });
-        it('Add and read record', async function() {
-            this.timeout(120000);
-            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
-            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
-            assert(added_record['test_add'] === 'test');
-            await db_api.removeRecord('test', {test_add: 'test'});
-        });
 
-        it('Find duplicates by key', async function() {
+        for (const local_db_mode of local_db_modes) {
-            const test_duplicates = [
+            let use_local_db = local_db_mode;
-                {
+            describe(`Use local DB - ${use_local_db}`, async function() {
-                    test: 'testing',
+                beforeEach(async function() {
-                    key: '1'
+                    if (!use_local_db) {
-                },
+                        this.timeout(120000);
-                {
+                        await db_api.connectToDB(0);
-                    test: 'testing',
+                    }
-                    key: '2'
+                    await db_api.removeAllRecords('test');
-                },
-                {
-                    test: 'testing_missing',
-                    key: '3'
-                },
-                {
-                    test: 'testing',
-                    key: '4'
-                }
-            ];
-            await db_api.insertRecordsIntoTable('test', test_duplicates);
-            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-            console.log(duplicates);
-        });
-
-        it('Update record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
-            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
-            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
-            assert(updated_record['added_field']);
-            await db_api.removeRecord('test', {test_update: 'test'});
-        });
-
-        it('Remove record', async function() {
-            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
-            assert(delete_succeeded);
-            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
-            assert(!deleted_record);
-        });
-
-        it('Push to record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
-            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 1);
-        });
-
-        it('Pull from record array', async function() {
-            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
-            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-            const record = await db_api.getRecord('test', {test: 'test'});
-            assert(record);
-            assert(record['test_array'].length === 0);
-        });
-
-        it('Bulk add', async function() {
-            this.timeout(120000);
-            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
-            const test_records = [];
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                test_records.push({
-                    uid: uuid()
                 });
-            }
+                it('Add and read record', async function() {
-            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    this.timeout(120000);
-
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
-            const received_records = await db_api.getRecords('test');
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
-            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                    assert(added_record['test_add'] === 'test');
-        });
+                    await db_api.removeRecord('test', {test_add: 'test'});
-
+                });
-        it('Bulk update', async function() {
+                it('Add and read record - Nested property', async function() {
-            // bulk add records
+                    this.timeout(120000);
-            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
-            const test_records = [];
+                    const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
-            const update_obj = {};
+                    const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                    assert(added_record['test_add'] === 'test');
-                const test_uid =  uuid();
+                    assert(!not_added_record);
-                test_records.push({
+                    await db_api.removeRecord('test', {test_add: 'test'});
-                    uid: test_uid
+                });
+                it('Replace filter', async function() {
+                    this.timeout(120000);
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+                    const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
+                    assert(count === 1);
+                    await db_api.removeRecord('test', {test_replace_filter: 'test'});
+                });
+                it('Find duplicates by key', async function() {
+                    const test_duplicates = [
+                        {
+                            test: 'testing',
+                            key: '1'
+                        },
+                        {
+                            test: 'testing',
+                            key: '2'
+                        },
+                        {
+                            test: 'testing_missing',
+                            key: '3'
+                        },
+                        {
+                            test: 'testing',
+                            key: '4'
+                        }
+                    ];
+                    await db_api.insertRecordsIntoTable('test', test_duplicates);
+                    const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+                    console.log(duplicates);
                 });
-                update_obj[test_uid] = {added_field: true};
-            }
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            assert(success);
 
-            // makes sure they are added
+                it('Update record', async function() {
-            const received_records = await db_api.getRecords('test');
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test'});
-            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                    await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+                    const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+                    assert(updated_record['added_field']);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
+                it('Update records', async function() {
-            assert(success);
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
+                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
+                    await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
+                    const updated_records = await db_api.getRecords('test', {added_field: true});
+                    assert(updated_records.length === 2);
+                    await db_api.removeRecord('test', {test_update: 'test'});
+                });
 
-            const received_updated_records = await db_api.getRecords('test');
+                it('Remove property from record', async function() {
-            for (let i = 0; i < received_updated_records.length; i++) {
+                    await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
-                success &= received_updated_records[i]['added_field'];
+                    await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
-            }
+                    const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
-            assert(success);
+                    assert(updated_record['test_keep']);
-        });
+                    assert(!updated_record['test_remove']);
+                    await db_api.removeRecord('test', {test_keep: 'test'});
+                });
 
-        it('Stats', async function() {
+                it('Remove record', async function() {
-            const stats = await db_api.getDBStats();
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            assert(stats);
+                    const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
-        });
+                    assert(delete_succeeded);
+                    const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+                    assert(!deleted_record);
+                });
 
-        it('Query speed', async function() {
+                it('Remove records', async function() {
-            this.timeout(120000); 
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
-            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
-            const test_records = [];
+                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
-            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+                    const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
-            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                    assert(delete_succeeded);
-                const uid = uuid();
+                    const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
-                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                    assert(count === 0);
-                test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J  - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon:  http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                });
-            }
-            const insert_start = Date.now();
-            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-            const insert_end = Date.now();
 
-            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+                it('Push to record array', async function() {
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+                    await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 1);
+                });
 
-            const query_start = Date.now();
+                it('Pull from record array', async function() {
-            const random_record = await db_api.getRecord('test', {uid: random_uid});
+                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
-            const query_end = Date.now();
+                    await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+                    const record = await db_api.getRecord('test', {test: 'test'});
+                    assert(record);
+                    assert(record['test_array'].length === 0);
+                });
 
-            console.log(random_record)
+                it('Bulk add', async function() {
+                    this.timeout(120000);
+                    const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+                    const test_records = [];
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        test_records.push({
+                            uid: uuid()
+                        });
+                    }
+                    const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
 
-            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+                    const received_records = await db_api.getRecords('test');
+                    assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+                });
 
-            success = !!random_record;
+                it('Bulk update', async function() {
+                    // bulk add records
+                    const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+                    const test_records = [];
+                    const update_obj = {};
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const test_uid =  uuid();
+                        test_records.push({
+                            uid: test_uid
+                        });
+                        update_obj[test_uid] = {added_field: true};
+                    }
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    assert(success);
 
-            assert(success);
+                    // makes sure they are added
-        });
+                    const received_records = await db_api.getRecords('test');
+                    assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+                    success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
+                    assert(success);
+
+                    const received_updated_records = await db_api.getRecords('test');
+                    for (let i = 0; i < received_updated_records.length; i++) {
+                        success &= received_updated_records[i]['added_field'];
+                    }
+                    assert(success);
+                });
+
+                it('Stats', async function() {
+                    const stats = await db_api.getDBStats();
+                    assert(stats);
+                });
+
+                it('Query speed', async function() {
+                    this.timeout(120000); 
+                    const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+                    const test_records = [];
+                    let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                        const uid = uuid();
+                        if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                        test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                    }
+                    const insert_start = Date.now();
+                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+                    const insert_end = Date.now();
+
+                    console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+                    const query_start = Date.now();
+                    const random_record = await db_api.getRecord('test', {uid: random_uid});
+                    const query_end = Date.now();
+
+                    console.log(random_record)
+
+                    console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+                    success = !!random_record;
+
+                    assert(success);
+                });
+            });
+        }
     });
+
+    describe('Local DB Filters', async function() {
+        it('Basic', async function() {
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: 'test'}, 'find');
+            assert(result && result['test'] === 'test');
+        });
+
+        it('Regex', async function() {
+            const filter = {$regex: `\\w+\\d`, $options: 'i'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Not equals', async function() {
+            const filter = {$ne: 'test'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Nested', async function() {
+            const result = db_api.applyFilterLocalDB([{test1: {test2: 'test3'}}, {test4: 'test5'}], {'test1.test2': 'test3'}, 'find');
+            assert(result && result['test1']['test2'] === 'test3');
+        });
+    })
 });
 
 describe('Multi User', async function() {
@@ -244,8 +343,6 @@ describe('Multi User', async function() {
     const playlist_to_test = 'ysabVZz4x';
     beforeEach(async function() {
         await db_api.connectToDB();
-        auth_api.initialize(db_api, logger);
-        subscriptions_api.initialize(db_api, logger);
         user = await auth_api.login('admin', 'pass');
     });
     describe('Authentication', function() {
@@ -253,10 +350,14 @@ describe('Multi User', async function() {
             assert(user);
         });
     });
-    describe('Video player - normal', function() {
+    describe('Video player - normal', async function() {
-        const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
+        beforeEach(async function() {
+            await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+            await db_api.insertRecordIntoTable('files', sample_video_json);
+        });
+        const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
-            const video_obj = db_api.getVideo(video_to_test, 'admin');
+            const video_obj = await files_api.getVideo(video_to_test);
             assert(video_obj);
         });
 
@@ -274,12 +375,12 @@ describe('Multi User', async function() {
     });
     describe('Zip generators', function() {
         it('Playlist zip generator', async function() {
-            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
+            const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
             assert(playlist);
             const playlist_files_to_download = [];
             for (let i = 0; i < playlist['uids'].length; i++) {
                 const uid = playlist['uids'][i];
-                const playlist_file = await db_api.getVideo(uid, user_to_test);
+                const playlist_file = await files_api.getVideo(uid, user_to_test);
                 playlist_files_to_download.push(playlist_file);
             }
             const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
@@ -307,7 +408,7 @@ describe('Multi User', async function() {
     //     const sub_to_test = '';
     //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
     //     it('Get video', async function() {
-    //         const video_obj = db_api.getVideo(video_to_test, 'admin', );
+    //         const video_obj = files_api.getVideo(video_to_test, 'admin', );
     //         assert(video_obj);
     //     });
 
@@ -341,7 +442,9 @@ describe('Downloader', function() {
     });
 
     it('Get file info', async function() {
-
+        this.timeout(300000);
+        const info = await downloader_api.getVideoInfoByURL(url);
+        assert(!!info);
     });
 
     it('Download file', async function() {
@@ -352,6 +455,19 @@ describe('Downloader', function() {
 
     });
 
+    it('Tag file', async function() {
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        const tags = {
+            title: sample_json['title'],
+            artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
+            TRCK: '27'
+        }
+        NodeID3.write(tags, audio_path);
+        const written_tags = NodeID3.read(audio_path);
+        assert(written_tags['raw']['TRCK'] === '27');
+    });
+
     it('Queue file', async function() {
         this.timeout(300000); 
         const returned_download = await downloader_api.createDownload(url, 'video', options);
@@ -360,20 +476,23 @@ describe('Downloader', function() {
     });
 
     it('Pause file', async function() {
-
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        await downloader_api.pauseDownload(returned_download['uid']);
+        const updated_download = await db_api.getRecord('download_queue', {uid: returned_download['uid']});
+        assert(updated_download['paused'] && !updated_download['running']);
     });
 
     it('Generate args', async function() {
         const args = await downloader_api.generateArgs(url, 'video', options);
-        console.log(args);
+        assert(args.length > 0);
     });
 
     it('Generate args - subscription', async function() {
-        subscriptions_api.initialize(db_api, logger);
         const sub = await subscriptions_api.getSubscription(sub_id);
         const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
-        const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        const args_normal = await downloader_api.generateArgs(url, 'video', options);
-        console.log(args);
+        const args_sub = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        console.log(JSON.stringify(args_normal) !== JSON.stringify(args_sub));
     });
 
     it('Generate kodi NFO file', async function() {
@@ -386,6 +505,40 @@ describe('Downloader', function() {
         assert(fs.existsSync(nfo_file_path), true);
         fs.unlinkSync(nfo_file_path);
     });
+
+    it('Inject args', async function() {
+        const original_args1 = ['--no-resize-buffer', '-o', '%(title)s', '--no-mtime'];
+        const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        const updated_args1 = utils.injectArgs(original_args1, new_args1);
+        const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
+
+        const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
+        const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        const updated_args2 = utils.injectArgs(original_args2, new_args2);
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
+
+        const original_args3 = ['-o', '%(title)s.%(ext)s'];
+        const new_args3 =  ['--min-filesize','1'];
+        const updated_args3 = utils.injectArgs(original_args3, new_args3);
+        const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
+        assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
+    });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1710641401';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
+    });
 });
 
 describe('Tasks', function() {
@@ -402,7 +555,7 @@ describe('Tasks', function() {
         };
         tasks_api.TASKS['dummy_task'] = dummy_task;
 
-        await tasks_api.initialize();
+        await tasks_api.setupTasks();
     });
     it('Backup db', async function() {
         const backups_original = await utils.recFindByExt('appdata', 'bak');
@@ -414,12 +567,13 @@ describe('Tasks', function() {
     });
 
     it('Check for missing files', async function() {
+        this.timeout(300000);
         await db_api.removeAllRecords('files', {uid: 'test'});
         const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
         await db_api.insertRecordIntoTable('files', test_missing_file);
         await tasks_api.executeTask('missing_files_check');
-        const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
+        const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
-        assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
+        assert(!missing_file_db_record, true);
     });
 
     it('Check for duplicate files', async function() {
@@ -432,10 +586,13 @@ describe('Tasks', function() {
         await db_api.insertRecordIntoTable('files', test_duplicate_file1);
         await db_api.insertRecordIntoTable('files', test_duplicate_file2);
         await db_api.insertRecordIntoTable('files', test_duplicate_file3);
-        await tasks_api.executeTask('duplicate_files_check');
+
+        await tasks_api.executeRun('duplicate_files_check');
         const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
-        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
         assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
+
+        await tasks_api.executeTask('duplicate_files_check');
+        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
         assert(duplicated_record_count == 1, true);
     });
 
@@ -452,7 +609,7 @@ describe('Tasks', function() {
         fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file, true);
+        assert(!!imported_file === true);
         
         // post-test cleanup
         if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -460,22 +617,239 @@ describe('Tasks', function() {
     });
 
     it('Schedule and cancel task', async function() {
-        const today_4_hours = new Date();
+        this.timeout(5000);
-        today_4_hours.setHours(today_4_hours.getHours() + 4);
+        const today_one_year = new Date();
-        await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
+        today_one_year.setFullYear(today_one_year.getFullYear() + 1);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+        const schedule_obj = {
+            type: 'timestamp',
+            data: { timestamp: today_one_year.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        const dummy_task = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
+        assert(!!dummy_task['schedule']);
+
         await tasks_api.updateTaskSchedule('dummy_task', null);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], false);
+        const dummy_task_updated = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!tasks_api.TASKS['dummy_task']['job']);
+        assert(!dummy_task_updated['schedule']);
     });
 
     it('Schedule and run task', async function() {
         this.timeout(5000);
         const today_1_second = new Date();
         today_1_second.setSeconds(today_1_second.getSeconds() + 1);
-        await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
+        const schedule_obj = {
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+            type: 'timestamp',
+            data: { timestamp: today_1_second.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
         await utils.wait(2000);
         const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
-        assert(dummy_task_obj['data'], true);
+        assert(dummy_task_obj['data']);
+    });
+});
+
+describe('Archive', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+    });
+
+    it('Import archive', async function() {
+        const archive_text = `
+            testextractor1 testing1
+            testextractor1 testing2
+            testextractor2 testing1
+            testextractor1 testing3
+
+        `;
+        const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
+        assert(count === 4)
+        const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        console.log(archive_items);
+        assert(archive_items.length === 4);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
+
+        const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        assert(success);
+    });
+
+    it('Get archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+
+        assert(archive_item1 && archive_item2);
+    });
+
+    it('Archive duplicates', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
+
+        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
+        assert(count === 3);
+    });
+
+    it('Remove from archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
+
+        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
+        assert(success);
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        assert(!!archive_item1);
+        
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+        assert(!archive_item2);
+
+        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
+        assert(!!archive_item3);
+    });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
+    });
+
+    it('Convert flat object to nested object', async function() {
+        // No modfication
+        const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
+        const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
+        assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
+        assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
+
+        // Standard setup
+        const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
+        const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
+        assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
+        assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
+
+        // Nested branches
+        const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
+        const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
+        assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
+    });
+});
+
+describe('Categories', async function() {
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        const new_category = {
+            name: 'test_category',
+            uid: uuid(),
+            rules: [],
+            custom_output: ''
+        };
+    
+        await db_api.insertRecordIntoTable('categories', new_category);
+    });
+
+    afterEach(async function() {
+        await db_api.removeAllRecords('categories', {name: 'test_category'});
+    });
+
+    it('Categorize - includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not includes', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        console.log(category);
+        assert(category && category.name === 'test_category');
+    });
+
+    it('Categorize - not equals', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'not_equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - AND', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'and',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(!category);
+    });
+
+    it('Categorize - OR', async function() {
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: null,
+            comparator: 'equals',
+            property: 'uploader',
+            value: 'Sample Uploader'
+        });
+
+        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
+            preceding_operator: 'or',
+            comparator: 'not_includes',
+            property: 'title',
+            value: 'Sample'
+        });
+
+        const category = await categories_api.categorize([sample_video_json]);
+        assert(category);
     });
 });

backend/twitch.js

@@ -1,90 +1,73 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
 const config_api = require('./config');
+const logger = require('./logger');
 
-async function getCommentsForVOD(clientID, vodId) {
+const moment = require('moment');
-    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
+const fs = require('fs-extra')
-        batch,
+const path = require('path');
-        cursor;
+const { promisify } = require('util');
+const child_process = require('child_process');
 
-    let comments = null;
+async function getCommentsForVOD(vodId) {
-
+    const exec = promisify(child_process.exec);
-    try {
+    
-        do {
+    // Reject invalid params to prevent command injection attack
-            batch = (await Axios.get(url, {
+    if (!vodId.match(/^[0-9a-z]+$/)) {
-                headers: {
+        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
-                    'Client-ID': clientID,
+        return null;
-                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
-                    'Content-Type': 'application/json; charset=UTF-8',
-                }
-            })).data;
-
-            const str = batch.comments.map(c => {
-                let {
-                        created_at: msgCreated,
-                        content_offset_seconds: timestamp,
-                        commenter: {
-                            name,
-                            _id,
-                            created_at: acctCreated
-                        },
-                        message: {
-                            body: msg,
-                            user_color: user_color
-                        }
-                    } = c;
-
-                const timestamp_str = moment.duration(timestamp, 'seconds')
-                    .toISOString()
-                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
-                        (_, ...ms) => {
-                            const seg = v => v ? v.padStart(2, '0') : '00';
-                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
-                        });
-
-                acctCreated = moment(acctCreated).utc();
-                msgCreated = moment(msgCreated).utc();
-
-                if (!comments) comments = [];
-
-                comments.push({
-                    timestamp: timestamp,
-                    timestamp_str: timestamp_str,
-                    name: name,
-                    message: msg,
-                    user_color: user_color
-                });
-                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
-                // return line;
-            }).join('\n');
-
-            cursor = batch._next;
-            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
-            await new Promise(res => setTimeout(res, 300));
-        } while (cursor);
-    } catch (err) {
-        console.error(err);
     }
 
-    return comments;
+    const is_windows = process.platform === 'win32';
+    const cliExt = is_windows ? '.exe' : ''
+    const cliPath = `TwitchDownloaderCLI${cliExt}`
+
+    if (!fs.existsSync(cliPath)) {
+        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
+        return null;
+    }
+
+    const result = await exec(`${cliPath} chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
+
+    if (result['stderr']) {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
+        logger.error(result['stderr']);
+        return null;
+    }
+
+    const temp_chat_path = path.join('appdata', `${vodId}.json`);
+
+    const raw_json = fs.readJSONSync(temp_chat_path);
+    const new_json = raw_json.comments.map(comment_obj => {
+        return {
+            timestamp: comment_obj.content_offset_seconds,
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+            name: comment_obj.commenter.name,
+            message: comment_obj.message.body,
+            user_color: comment_obj.message.user_color
+        }
+    });
+
+    fs.unlinkSync(temp_chat_path);
+
+    return new_json;
 }
 
 async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
     let file_path = null;
 
     if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
         }
     }
 
@@ -96,23 +79,26 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
     return chat_file;
 }
 
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
-    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
-    const chat = await getCommentsForVOD(twitch_api_key, vodId);
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const chat = await getCommentsForVOD(vodId);
 
     // save file if needed params are included
     let file_path = null;
-    if (user_uid) {
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            file_path = path.join(type, `${id}.twitch_chat.json`);
         }
     }
 
@@ -121,6 +107,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
     return chat;
 }
 
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
 module.exports = {
     getCommentsForVOD: getCommentsForVOD,
     getTwitchChatByFileID: getTwitchChatByFileID,

backend/utils.js

@@ -4,6 +4,7 @@ const ffmpeg = require('fluent-ffmpeg');
 const archiver = require('archiver');
 const fetch = require('node-fetch');
 const ProgressBar = require('progress');
+const winston = require('winston');
 
 const config_api = require('./config');
 const logger = require('./logger');
@@ -12,7 +13,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-function getTrueFileName(unfixed_path, type) {
+exports.getTrueFileName = (unfixed_path, type) => {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -27,13 +28,13 @@ function getTrueFileName(unfixed_path, type) {
     return fixed_path;
 }
 
-async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
+exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
     // return empty array if the path doesn't exist
     if (!(await fs.pathExists(basePath))) return [];
 
     let files = [];
     const ext = type === 'audio' ? 'mp3' : 'mp4';
-    var located_files = await recFindByExt(basePath, ext);
+    var located_files = await exports.recFindByExt(basePath, ext);
     for (let i = 0; i < located_files.length; i++) {
         let file = located_files[i];
         var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -41,33 +42,33 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
         var stats = await fs.stat(file);
 
         var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await getJSONByType(type, id, basePath);
+        var jsonobj = await exports.getJSONByType(type, id, basePath);
         if (!jsonobj) continue;
         if (full_metadata) {
             jsonobj['id'] = id;
             files.push(jsonobj);
             continue;
         }
-        var upload_date = formatDateString(jsonobj.upload_date);
+        var upload_date = exports.formatDateString(jsonobj.upload_date);
 
         var isaudio = type === 'audio';
-        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+        var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
                                 stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
 }
 
-async function createContainerZipFile(file_name, container_file_objs) {
+exports.createContainerZipFile = async (file_name, container_file_objs) => {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
 }
 
-async function createZipFile(zip_file_path, file_paths) {
+exports.createZipFile = async (zip_file_path, file_paths) => {
     let output = fs.createWriteStream(zip_file_path);
 
     var archive = archiver('zip', {
@@ -91,11 +92,11 @@ async function createZipFile(zip_file_path, file_paths) {
     await archive.finalize();
 
     // wait a tiny bit for the zip to reload in fs
-    await wait(100);
+    await exports.wait(100);
     return zip_file_path;
 }
 
-function getJSONMp4(name, customPath, openReadPerms = false) {
+exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -110,7 +111,7 @@ function getJSONMp4(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSONMp3(name, customPath, openReadPerms = false) {
+exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
     var obj = null;
     if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -127,11 +128,11 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
-function getJSON(file_path, type) {
+exports.getJSON = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
     let obj = null;
-    var jsonPath = removeFileExtension(file_path) + '.info.json';
+    var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
-    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
     if (fs.existsSync(jsonPath))
     {
         obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -142,12 +143,12 @@ function getJSON(file_path, type) {
     return obj;
 }
 
-function getJSONByType(type, name, customPath, openReadPerms = false) {
+exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
-    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+    return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(file_path) {
+exports.getDownloadedThumbnail = (file_path) => {
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
     let webpPath = file_path_no_extension + '.webp';
@@ -163,7 +164,7 @@ function getDownloadedThumbnail(file_path) {
         return null;
 }
 
-function getExpectedFileSize(input_info_jsons) {
+exports.getExpectedFileSize = (input_info_jsons) => {
     // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
     const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
 
@@ -172,11 +173,13 @@ function getExpectedFileSize(input_info_jsons) {
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
-            info_json.formats.forEach(available_format => {
+            if (info_json.formats !== undefined) {
-                if (available_format.format_id === format_id && available_format.filesize) {
+                info_json.formats.forEach(available_format => {
-                    individual_expected_filesize += available_format.filesize;
+                  if (available_format.format_id === format_id && (available_format.filesize || available_format.filesize_approx)) {
-                }
+                    individual_expected_filesize += (available_format.filesize ? available_format.filesize : available_format.filesize_approx);
-            });
+                  }
+                });
+            }
         });
         expected_filesize += individual_expected_filesize;
     });
@@ -184,12 +187,12 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(file_path, type) {
+exports.fixVideoMetadataPerms = (file_path, type) => {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
 
     const files_to_fix = [
         // JSONs
@@ -206,11 +209,11 @@ function fixVideoMetadataPerms(file_path, type) {
     }
 }
 
-function deleteJSONFile(file_path, type) {
+exports.deleteJSONFile = (file_path, type) => {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = removeFileExtension(file_path);
+    const file_path_no_extension = exports.removeFileExtension(file_path);
-    
+
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
 
@@ -218,33 +221,7 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-async function removeIDFromArchive(archive_path, id) {
+exports.durationStringToNumber = (dur_str) => {
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
-    if (!data) {
-        logger.error('Archive could not be found.');
-        return;
-    }
-
-    let dataArray = data.split('\n'); // convert file data in an array
-    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-    let lastIndex = -1; // let say, we have not found the keyword
-
-    for (let index=0; index<dataArray.length; index++) {
-        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-            lastIndex = index; // found a line includes a id keyword
-            break;
-        }
-    }
-
-    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-
-    // UPDATE FILE WITH NEW DATA
-    const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
-    if (line) return line;
-}
-
-function durationStringToNumber(dur_str) {
     if (typeof dur_str === 'number') return dur_str;
     let num_sum = 0;
     const dur_str_parts = dur_str.split(':');
@@ -254,23 +231,22 @@ function durationStringToNumber(dur_str) {
     return num_sum;
 }
 
-function getMatchingCategoryFiles(category, files) {
+exports.getMatchingCategoryFiles = (category, files) => {
     return files && files.filter(file => file.category && file.category.uid === category.uid);
 }
 
-function addUIDsToCategory(category, files) {
+exports.addUIDsToCategory = (category, files) => {
-    const files_that_match = getMatchingCategoryFiles(category, files);
+    const files_that_match = exports.getMatchingCategoryFiles(category, files);
     category['uids'] = files_that_match.map(file => file.uid);
     return files_that_match;
 }
 
-function getCurrentDownloader() {
+exports.getCurrentDownloader = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     return details_json['downloader'];
 }
 
-async function recFindByExt(base, ext, files, result, recursive = true)
+exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
-{
     files = files || (await fs.readdir(base))
     result = result || []
 
@@ -279,7 +255,7 @@ async function recFindByExt(base, ext, files, result, recursive = true)
         if ( (await fs.stat(newbase)).isDirectory() )
         {
             if (!recursive) continue;
-            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+            result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
         {
@@ -292,23 +268,23 @@ async function recFindByExt(base, ext, files, result, recursive = true)
     return result
 }
 
-function removeFileExtension(filename) {
+exports.removeFileExtension = (filename) => {
     const filename_parts = filename.split('.');
     filename_parts.splice(filename_parts.length - 1);
     return filename_parts.join('.');
 }
 
-function formatDateString(date_string) {
+exports.formatDateString = (date_string) => {
     return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
 }
 
-function createEdgeNGrams(str) {
+exports.createEdgeNGrams = (str) => {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
-        
+
         return str.split(" ").reduce((ngrams, token) => {
-            if (token.length > minGram) {   
+            if (token.length > minGram) {
                 for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
                     ngrams = [...ngrams, token.substr(0, i)]
                 }
@@ -318,13 +294,13 @@ function createEdgeNGrams(str) {
             return ngrams
         }, []).join(" ")
     }
-    
+
     return str
 }
 
 // ffmpeg helper functions
 
-async function cropFile(file_path, start, end, ext) {
+exports.cropFile = async (file_path, start, end, ext) => {
     return new Promise(resolve => {
         const temp_file_path = `${file_path}.cropped${ext}`;
         let base_ffmpeg_call = ffmpeg(file_path);
@@ -353,13 +329,13 @@ async function cropFile(file_path, start, end, ext) {
  * setTimeout, but its a promise.
  * @param {number} ms
  */
- async function wait(ms) {
+exports.wait = async (ms) => {
     await new Promise(resolve => {
         setTimeout(resolve, ms);
     });
 }
 
-async function checkExistsWithTimeout(filePath, timeout) {
+exports.checkExistsWithTimeout = async (filePath, timeout) => {
     return new Promise(function (resolve, reject) {
 
         var timer = setTimeout(function () {
@@ -388,7 +364,7 @@ async function checkExistsWithTimeout(filePath, timeout) {
 }
 
 // helper function to download file using fetch
-async function fetchFile(url, path, file_label) {
+exports.fetchFile = async (url, path, file_label) => {
     var len = null;
     const res = await fetch(url);
 
@@ -415,6 +391,134 @@ async function fetchFile(url, path, file_label) {
     });
 }
 
+exports.restartServer = async (is_update = false) => {
+    logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
+
+    // the following line restarts the server through pm2
+    fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
+    process.exit(1);
+}
+
+// adds or replaces args according to the following rules:
+//  - if it already exists and has value, then replace both arg and value
+//  - if already exists and doesn't have value, ignore
+//  - if it doesn't exist and has value, add both arg and value
+//  - if it doesn't exist and doesn't have value, add arg
+exports.injectArgs = (original_args, new_args) => {
+    const updated_args = original_args.slice();
+    try {
+        for (let i = 0; i < new_args.length; i++) {
+            const new_arg = new_args[i];
+            if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
+
+            if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
+                if (original_args.includes(new_arg)) {
+                    const original_index = original_args.indexOf(new_arg);
+                    updated_args.splice(original_index, 2);
+                }
+
+                updated_args.push(new_arg, new_args[i + 1]);
+                i++; // we need to skip the arg value on the next loop
+            } else {
+                if (!original_args.includes(new_arg)) {
+                    updated_args.push(new_arg);
+                }
+            }
+        }
+    } catch (err) {
+        logger.warn(err);
+        logger.warn(`Failed to inject args (${new_args}) into (${original_args})`);
+    }
+
+    return updated_args;
+}
+
+exports.filterArgs = (args, args_to_remove) => {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+exports.searchObjectByString = (o, s) => {
+    s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
+    s = s.replace(/^\./, '');           // strip a leading dot
+    var a = s.split('.');
+    for (var i = 0, n = a.length; i < n; ++i) {
+        var k = a[i];
+        if (k in o) {
+            o = o[k];
+        } else {
+            return;
+        }
+    }
+    return o;
+}
+
+exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
+exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
+exports.getBaseURL = () => {
+    return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
+}
+
+exports.updateLoggerLevel = (new_logger_level) => {
+    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    if (!possible_levels.includes(new_logger_level)) {
+        logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
+        new_logger_level = 'info';
+    }
+    logger.level = new_logger_level;
+    winston.loggers.get('console').level = new_logger_level;
+    logger.transports[2].level = new_logger_level;
+}
+
+exports.convertFlatObjectToNestedObject = (obj) => {
+    const result = {};
+    for (const key in obj) {
+      const nestedKeys = key.split('.');
+      let currentObj = result;
+      for (let i = 0; i < nestedKeys.length; i++) {
+        if (i === nestedKeys.length - 1) {
+          currentObj[nestedKeys[i]] = obj[key];
+        } else {
+          currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
+          currentObj = currentObj[nestedKeys[i]];
+        }
+      }
+    }
+    return result;
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -432,31 +536,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.view_count = view_count;
     this.height = height;
     this.abr = abr;
-}
+    this.favorite = false;
+}   
+exports.File = File;
 
-module.exports = {
-    getJSONMp3: getJSONMp3,
-    getJSONMp4: getJSONMp4,
-    getJSON: getJSON,
-    getTrueFileName: getTrueFileName,
-    getDownloadedThumbnail: getDownloadedThumbnail,
-    getExpectedFileSize: getExpectedFileSize,
-    fixVideoMetadataPerms: fixVideoMetadataPerms,
-    deleteJSONFile: deleteJSONFile,
-    removeIDFromArchive: removeIDFromArchive,
-    getDownloadedFilesByType: getDownloadedFilesByType,
-    createContainerZipFile: createContainerZipFile,
-    durationStringToNumber: durationStringToNumber,
-    getMatchingCategoryFiles: getMatchingCategoryFiles,
-    addUIDsToCategory: addUIDsToCategory,
-    getCurrentDownloader: getCurrentDownloader,
-    recFindByExt: recFindByExt,
-    removeFileExtension: removeFileExtension,
-    formatDateString: formatDateString,
-    cropFile: cropFile,
-    createEdgeNGrams: createEdgeNGrams,
-    wait: wait,
-    checkExistsWithTimeout: checkExistsWithTimeout,
-    fetchFile: fetchFile,
-    File: File
-}

backend/youtube-dl.js

@@ -6,6 +6,8 @@ const utils = require('./utils');
 const CONSTS = require('./consts');
 const config_api = require('./config.js');
 
+const OUTDATED_VERSION = "2020.00.00";
+
 const is_windows = process.platform === 'win32';
 
 const download_sources = {
@@ -31,7 +33,7 @@ exports.checkForYoutubeDLUpdate = async () => {
         let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
         if (!current_app_details_exists) {
             logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
-            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
+            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader});
         }
         let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
         let current_version = current_app_details['version'];
@@ -86,6 +88,18 @@ exports.updateYoutubeDL = async (latest_update_version) => {
     await download_sources[default_downloader]['func'](latest_update_version);
 }
 
+exports.verifyBinaryExistsLinux = () => {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
+        details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
+        details_json['exec'] = 'youtube-dl';
+        details_json['version'] = OUTDATED_VERSION;
+        fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
+
+        utils.restartServer();
+    }
+}
+
 async function downloadLatestYoutubeDLBinary(new_version) {
     const file_ext = is_windows ? '.exe' : '';
 

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.2"
+appVersion: "4.3.1"

chrome-extension.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMX9Wk5SM5cIfY
-6ReKX3ybY1rsbNbOzG8ceN7yyeXB0mor8pVsX1MOna2HewOyBuaaYNJRO4tJBxic
-7a8zQErfgHL/i/QrVvVCpfJ7xKvq6zij5NYoqd/FBUwawqjeH5/voIcAp9z5Vmsr
-kL0sxJUKy6b4IWNp3noU7Nvq2RwxnXQbKDhz8FrX6oQAnDC6gsG5a2OSPsaE4oqw
-6nmonORJypmpP5hqyHY8ffXBT2lAxjHT7OnYbaCBe2TQP8+rH6rDBOhjVNtUJ089
-ocTQL6LtQEPkcF4yKJmtcOwHl8OPGZs5l9i8xb4j9RuSPkm2lbzZX8sOsdGGoqJZ
-q68nYhsHAgMBAAECggEAXmtKEzfPObq88B/kAcgSk+FngMHZzcmR7bgD3GwdSxnQ
-dkRI9zvk7eQ35tcUwntAr4Lat6/ILjFqlBmVLxrdXHuF5Xz9jcZLYgKzz61xdYM9
-dC6FKF0u5eGIIvbauGAo7jaeGFX1F3Zu5b4lP9kEOGwU1B7sxF0FzsQM5+dtCJgv
-We/hWQeF+9gtoVnkCSS/Mq2p0UomXXHW0Bz4+HuHlTR9aiYbviYnotABiLUhZyzt
-v5yUaktb9qniBfdLpRlq8cp06xYlTEA9gJpa4Pnok8OWUsbAiW6EiXUSaZ/cchVa
-AnO8WWYvVOnnt6WHI3+QdFTnqVjE5TBX4N/7bVhHGQKBgQD0dtbFqp7vZK/jVYvE
-z0WPdySOg2ZDmoSfk5ZlR1+Y9zWToHv0qu8zqoOjL8Ubxrh9fGlOow+cCVdkEuaa
-jWC2AWetuRvW0Z5A3XMXr0/N/1fgOkTqtp3WNrUPjVJahEg3lN+90opgFoT8swSi
-s1oxW0oLcVIlrjhGBXAPCfsAuQKBgQDWBLRhHsRAvGcK5wGuVnxVApTIyBOermsW
-3bJt+7+UI+4sYrBAwkWdQG93IG0cQtn48TEPBgmR2fjRF5IFT9M4/u+QOeeByT7I
-we7nVtHgSY5ByC9N0mjWbcmSg8fktz/LonjldNC4kWdOFb75fxGf8kOGS5rUaMA4
-zHucfB6ZvwKBgQCPHJrysMXGY21MaqIeHzEboaX3ABl37hdBzAa5V6UxSVdGCydF
-vmO2HVZey/JaJmWOoKyNaowSzq0oWqBBTg6VvhDR9JHFmoVId9uOvAS+FYN+Mt5x
-gWK5KuGoLxVNBC+6yh6JY526TrSfsrU+Aj0Es+qO9FIg2PL8muZVB4S3kQKBgH/5
-CDMaxpc/EQ5/2413wZjDllwI51J3USm3Hz6Mzp2ybnSz/lh60k2Zfg1polTH1Lb6
-4i7tmUNRZ2sAARyUAuWN64n+VeRRhe1dqZFDZPQMh7fmEAMk0fOGaoXlrt2ghdEq
-Mchi9Xun1nHmpu9hgBR4NNBU3RwuFuLfwvprbZDZAoGAWa62QJChE86xQGP1MrL2
-SbIzw3cfeP5xdQ3MKldJiy5IkbMR7Z13WZ7FwvPTy0g/onLHD1rqlm1kUMsGRHpD
-5vH06PNpKXQ6x8BYaRGtE6P39jLycO/X+WK/lYTrWo1bR+mGCebDh4B5XrwT3gI6
-x4Gvz134pZCTyQCf5JCwbQs=
------END PRIVATE KEY-----

docker-build.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-
-# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
-# and also optimizing some code with this commit.
-# xoxo :D
-
-set -xeuo pipefail
-
-case $(uname -m) in
-  x86_64)
-    ARCH=amd64;;
-  aarch64)
-    ARCH=arm64;;
-  armhf)
-    ARCH=armhf;;
-  armv7)
-    ARCH=armel;;
-  armv7l)
-    ARCH=armel;;
-  *)
-    echo "Unsupported architecture: $(uname -m)"
-    exit 1
-esac
-
-echo "Architecture: $ARCH"
-wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
-mkdir /tmp/ffmpeg
-tar xf ffmpeg.txz -C /tmp/ffmpeg
-cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
-cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
-rm -rf /tmp/ffmpeg ffmpeg.txz

docker-compose.yml

@@ -2,7 +2,6 @@ version: "2"
 services:
     ytdl_material:
         environment: 
-            ALLOW_CONFIG_MUTATIONS: 'true'
             ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
             ytdl_use_local_db: 'false'
             write_ytdl_config: 'true'
@@ -17,14 +16,13 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:nightly
+        image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        image: mongo
+        # If you are using a Raspberry Pi, use mongo:4.4.18
-        ports:
+        image: mongo:4
-            - "27017:27017"
         logging:
             driver: "none"          
         container_name: mongo-db
         restart: always
         volumes:
-            - ./db/:/data/db
+            - ./db/:/data/db

docker-utils/GetTwitchDownloader.py

@@ -0,0 +1,69 @@
+import platform
+import requests
+import shutil
+import os
+import re
+import sys
+from collections import OrderedDict
+
+from github import Github
+
+machine = platform.machine()
+
+# https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
+MACHINES_TO_ZIP = OrderedDict([
+    ("x86_64", "Linux-x64"),
+    ("aarch64", "LinuxArm64"),
+    ("armv8", "LinuxArm64"),
+    ("arm", "LinuxArm"),
+    ("AMD64", "Windows-x64")
+])
+
+def getZipName():
+    for possibleMachine, possibleZipName in MACHINES_TO_ZIP.items():
+        if possibleMachine in machine:
+            return possibleZipName
+
+def getLatestFileInRepo(repo, search_string):
+    # Create an unauthenticated instance of the Github object
+    g = Github(os.environ.get('GH_TOKEN'))
+
+    # Replace with the repository owner and name
+    repo = g.get_repo(repo)
+
+    # Get all releases of the repository
+    releases = repo.get_releases()
+
+    # Loop through the releases in reverse order (from latest to oldest)
+    for release in list(releases):
+        # Get the release assets (files attached to the release)
+        assets = release.get_assets()
+
+        # Loop through the assets
+        for asset in assets:
+            if re.search(search_string, asset.name):
+                print(f'Downloading: {asset.name}')
+                response = requests.get(asset.browser_download_url)
+                with open(asset.name, 'wb') as f:
+                    f.write(response.content)
+                print(f'Download complete: {asset.name}. Unzipping...')
+                shutil.unpack_archive(asset.name, './')
+                print(f'Unzipping complete!')
+                os.remove(asset.name)
+                break
+        else:
+            continue
+        break
+    else:
+        # If no matching release is found, print a message
+        print(f'No release found with {search_string}')
+
+def getLatestCLIRelease():
+    zipName = getZipName()
+    if not zipName:
+        print(f"GetTwitchDownloader.py could not get valid path for '{machine}'. Exiting...")
+        sys.exit(1)
+    searchString = r'.*CLI.*' + zipName
+    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
+
+getLatestCLIRelease()

docker-utils/ffmpeg-fetch.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
+# and also optimizing some code with this commit.
+# xoxo :D
+
+case $(uname -m) in
+  x86_64)
+    ARCH=amd64;;
+  aarch64)
+    ARCH=arm64;;
+  armhf)
+    ARCH=armhf;;
+  armv7)
+    ARCH=armel;;
+  armv7l)
+    ARCH=armel;;
+  *)
+    echo "Unsupported architecture: $(uname -m)"
+    exit 1
+esac
+
+echo "(INFO) Architecture detected: $ARCH"
+echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
+apt-get update && apt-get -y install curl xz-utils
+echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
+curl -o ffmpeg.txz \
+    --connect-timeout 5 \
+    --max-time 120 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
+mkdir /tmp/ffmpeg
+tar xf ffmpeg.txz -C /tmp/ffmpeg
+echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
+apt-get -y remove curl xz-utils
+apt-get -y autoremove
+echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
+cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
+cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
+echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
+rm -rf /tmp/ffmpeg ffmpeg.txz

heroku.yml

@@ -0,0 +1,3 @@
+build:
+  docker:
+    web: Dockerfile.heroku

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.2.0",
+  "version": "4.3.1",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -12,7 +12,8 @@
     "lint": "ng lint",
     "e2e": "ng e2e",
     "electron": "ng build --base-href ./ && electron .",
-    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
+    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n --out-file=messages.en.xlf"
   },
   "engines": {
     "node": "12.3.1",
@@ -20,62 +21,60 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
+    "@angular-devkit/core": "^15.0.1",
-    "@angular/animations": "^13.3.4",
+    "@angular/animations": "^15.0.1",
-    "@angular/cdk": "^13.3.4",
+    "@angular/cdk": "^15.0.0",
-    "@angular/common": "^13.3.4",
+    "@angular/common": "^15.0.1",
-    "@angular/compiler": "^13.3.4",
+    "@angular/compiler": "^15.0.1",
-    "@angular/core": "^13.3.4",
+    "@angular/core": "^15.0.1",
-    "@angular/forms": "^13.3.4",
+    "@angular/forms": "^15.0.1",
-    "@angular/localize": "^13.3.4",
+    "@angular/localize": "^15.0.1",
-    "@angular/material": "^13.3.4",
+    "@angular/material": "^15.0.0",
-    "@angular/platform-browser": "^13.3.4",
+    "@angular/platform-browser": "^15.0.1",
-    "@angular/platform-browser-dynamic": "^13.3.4",
+    "@angular/platform-browser-dynamic": "^15.0.1",
-    "@angular/router": "^13.3.4",
+    "@angular/router": "^15.0.1",
     "@fontsource/material-icons": "^4.5.4",
-    "@ngneat/content-loader": "^5.0.0",
+    "@ngneat/content-loader": "^7.0.0",
-    "@videogular/ngx-videogular": "^5.0.1",
+    "@videogular/ngx-videogular": "^6.0.0",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
-    "filesize": "^6.1.0",
+    "filesize": "^10.0.7",
-    "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
-    "ng-lazyload-image": "^7.0.1",
+    "ngx-avatars": "^1.4.1",
-    "ngx-avatars": "^1.3.1",
+    "ngx-file-drop": "^15.0.0",
-    "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
-    "rxjs-compat": "^6.0.0-rc.0",
+    "rxjs-compat": "^6.6.7",
     "tslib": "^2.0.0",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "xliff-to-json": "^1.0.4",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
+    "@angular-devkit/build-angular": "^15.0.1",
-    "@angular/cli": "^13.3.3",
+    "@angular/cli": "^15.0.1",
-    "@angular/compiler-cli": "^13.3.4",
+    "@angular/compiler-cli": "^15.0.1",
-    "@angular/language-service": "^13.3.4",
+    "@angular/language-service": "^15.0.1",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "~3.6.0",
+    "@types/jasmine": "^4.3.1",
     "@types/node": "^12.11.1",
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
+    "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
-    "electron": "^13.6.6",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.3.16",
+    "karma": "~6.4.2",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
-    "karma-jasmine": "~4.0.0",
+    "karma-jasmine": "~5.1.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.21.0",
+    "openapi-typescript-codegen": "^0.23.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -3,7 +3,9 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
+export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
 export type { body_20 } from './models/body_20';
 export type { Category } from './models/Category';
@@ -12,6 +14,7 @@ export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermission
 export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
 export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
 export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
+export type { ClearDownloadsRequest } from './models/ClearDownloadsRequest';
 export type { ConcurrentStream } from './models/ConcurrentStream';
 export type { Config } from './models/Config';
 export type { ConfigResponse } from './models/ConfigResponse';
@@ -23,8 +26,11 @@ export type { CropFileSettings } from './models/CropFileSettings';
 export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
+export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
 export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
 export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -36,16 +42,19 @@ export type { DownloadResponse } from './models/DownloadResponse';
 export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
 export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
 export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
-export type { File } from './models/File';
 export { FileType } from './models/FileType';
+export { FileTypeFilter } from './models/FileTypeFilter';
 export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
 export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
 export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
 export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
 export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetArchivesRequest } from './models/GetArchivesRequest';
+export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -59,6 +68,7 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
 export type { GetLogsResponse } from './models/GetLogsResponse';
 export type { GetMp3sResponse } from './models/GetMp3sResponse';
 export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
 export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
 export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
 export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -69,17 +79,24 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
 export type { LoginResponse } from './models/LoginResponse';
+export type { Notification } from './models/Notification';
+export { NotificationAction } from './models/NotificationAction';
+export { NotificationType } from './models/NotificationType';
 export type { Playlist } from './models/Playlist';
 export type { RegisterRequest } from './models/RegisterRequest';
 export type { RegisterResponse } from './models/RegisterResponse';
+export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
 export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
 export type { SharingToggle } from './models/SharingToggle';
+export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
 export type { SubscribeResponse } from './models/SubscribeResponse';
 export type { Subscription } from './models/Subscription';
@@ -98,12 +115,15 @@ export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
 export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
 export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
 export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
+export type { UpdateFileRequest } from './models/UpdateFileRequest';
 export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';
 export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
+export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type AddFileToPlaylistRequest = {
-export interface AddFileToPlaylistRequest {
     file_uid: string;
     playlist_id: string;
-}
+};

src/api-types/models/Archive.ts

@@ -0,0 +1,16 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileType } from './FileType';
+
+export type Archive = {
+    extractor: string;
+    id: string;
+    type: FileType;
+    title: string;
+    user_uid?: string;
+    sub_id?: string;
+    timestamp: number;
+    uid: string;
+};

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { UserPermission } from './UserPermission';
+import type { UserPermission } from './UserPermission';
-import { YesNo } from './YesNo';
+import type { YesNo } from './YesNo';
 
-export interface BaseChangePermissionsRequest {
+export type BaseChangePermissionsRequest = {
     permission: UserPermission;
     new_value: YesNo;
-}
+};

src/api-types/models/Category.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { CategoryRule } from './CategoryRule';
+import type { CategoryRule } from './CategoryRule';
 
-export interface Category {
+export type Category = {
     name?: string;
     uid?: string;
     rules?: Array<CategoryRule>;
@@ -12,4 +12,4 @@ export interface Category {
      * Overrides file output for downloaded files in category
      */
     custom_output?: string;
-}
+};

src/api-types/models/CategoryRule.ts

@@ -2,11 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CategoryRule = {
-export interface CategoryRule {
     preceding_operator?: CategoryRule.preceding_operator;
     comparator?: CategoryRule.comparator;
-}
+};
 
 export namespace CategoryRule {
 

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
 
-export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
+export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
-    role: string;
+role: string;
-}
+});

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
 
-export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
+export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
-    user_uid: string;
+user_uid: string;
-}
+});

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -2,10 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CheckConcurrentStreamRequest = {
-export interface CheckConcurrentStreamRequest {
     /**
      * UID of the concurrent stream
      */
     uid: string;
-}
+};

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { ConcurrentStream } from './ConcurrentStream';
+import type { ConcurrentStream } from './ConcurrentStream';
 
-export interface CheckConcurrentStreamResponse {
+export type CheckConcurrentStreamResponse = {
     stream: ConcurrentStream;
-}
+};

src/api-types/models/ClearDownloadsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type ClearDownloadsRequest = {
+    clear_finished?: boolean;
+    clear_paused?: boolean;
+    clear_errors?: boolean;
+};

src/api-types/models/ConcurrentStream.ts

@@ -2,9 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type ConcurrentStream = {
-export interface ConcurrentStream {
     playback_timestamp?: number;
     unix_timestamp?: number;
     playing?: boolean;
-}
+};

src/api-types/models/Config.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type Config = {
-export interface Config {
     YoutubeDLMaterial: any;
-}
+};

src/api-types/models/ConfigResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Config } from './Config';
+import type { Config } from './Config';
 
-export interface ConfigResponse {
+export type ConfigResponse = {
     config_file: Config;
     success: boolean;
-}
+};

src/api-types/models/CreateCategoryRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CreateCategoryRequest = {
-export interface CreateCategoryRequest {
     name: string;
-}
+};

src/api-types/models/CreateCategoryResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Category } from './Category';
+import type { Category } from './Category';
 
-export interface CreateCategoryResponse {
+export type CreateCategoryResponse = {
     new_category?: Category;
     success?: boolean;
-}
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -2,11 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+export type CreatePlaylistRequest = {
-
-export interface CreatePlaylistRequest {
     playlistName: string;
     uids: Array<string>;
-    type: FileType;
     thumbnailURL: string;
-}
+};

src/api-types/models/CreatePlaylistResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface CreatePlaylistResponse {
+export type CreatePlaylistResponse = {
     new_playlist: Playlist;
     success: boolean;
-}
+};

src/api-types/models/CropFileSettings.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CropFileSettings = {
-export interface CropFileSettings {
     cropFileStart: number;
     cropFileEnd: number;
-}
+};

src/api-types/models/DBBackup.ts

@@ -2,13 +2,12 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DBBackup = {
-export interface DBBackup {
     name: string;
     timestamp: number;
     size: number;
     source: DBBackup.source;
-}
+};
 
 export namespace DBBackup {
 

src/api-types/models/DBInfoResponse.ts

@@ -2,17 +2,18 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { TableInfo } from './TableInfo';
+import type { TableInfo } from './TableInfo';
 
-export interface DBInfoResponse {
+export type DBInfoResponse = {
     using_local_db?: boolean;
     stats_by_table?: {
-files?: TableInfo,
+files?: TableInfo;
-playlists?: TableInfo,
+playlists?: TableInfo;
-categories?: TableInfo,
+categories?: TableInfo;
-subscriptions?: TableInfo,
+subscriptions?: TableInfo;
-users?: TableInfo,
+users?: TableInfo;
-roles?: TableInfo,
+roles?: TableInfo;
-download_queue?: TableInfo,
+download_queue?: TableInfo;
+archives?: TableInfo;
 };
-}
+};

src/api-types/models/DatabaseFile.ts

@@ -2,11 +2,16 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { Category } from './Category';
 
-export interface DatabaseFile {
+export type DatabaseFile = {
     id: string;
     title: string;
+    /**
+     * Backup if thumbnailPath is not defined
+     */
     thumbnailURL: string;
+    thumbnailPath?: string;
     isAudio: boolean;
     /**
      * In seconds
@@ -14,9 +19,27 @@ export interface DatabaseFile {
     duration: number;
     url: string;
     uploader: string;
+    /**
+     * In bytes
+     */
     size: number;
     path: string;
     upload_date: string;
     uid: string;
+    user_uid?: string;
     sharingEnabled?: boolean;
-}
+    category?: Category;
+    view_count?: number;
+    local_view_count?: number;
+    sub_id?: string;
+    registered?: number;
+    /**
+     * In pixels, only for videos
+     */
+    height?: number;
+    /**
+     * In Kbps
+     */
+    abr?: number;
+    favorite: boolean;
+};

src/api-types/models/DeleteAllFilesResponse.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteAllFilesResponse = {
+    /**
+     * Number of files found matching search parameters
+     */
+    file_count?: number;
+    /**
+     * Number of files removed
+     */
+    delete_count?: number;
+};

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { Archive } from './Archive';
+
+export type DeleteArchiveItemsRequest = {
+    archives: Array<Archive>;
+};

src/api-types/models/DeleteCategoryRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteCategoryRequest = {
-export interface DeleteCategoryRequest {
     category_uid: string;
-}
+};

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteMp3Mp4Request = {
-export interface DeleteMp3Mp4Request {
     uid: string;
     blacklistMode?: boolean;
-}
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteNotificationRequest = {
+    uid: string;
+};

src/api-types/models/DeletePlaylistRequest.ts

@@ -2,9 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+export type DeletePlaylistRequest = {
-
-export interface DeletePlaylistRequest {
     playlist_id: string;
-    type: FileType;
+};
-}

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,14 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { SubscriptionRequestData } from './SubscriptionRequestData';
+export type DeleteSubscriptionFileRequest = {
-
+    file_uid: string;
-export interface DeleteSubscriptionFileRequest {
-    file: string;
-    file_uid?: string;
-    sub: SubscriptionRequestData;
     /**
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */
     deleteForever?: boolean;
-}
+};

src/api-types/models/DeleteUserRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteUserRequest = {
-export interface DeleteUserRequest {
     uid: string;
-}
+};

src/api-types/models/Download.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type Download = {
-export interface Download {
     uid: string;
     ui_uid?: string;
     running: boolean;
@@ -20,7 +19,12 @@ export interface Download {
      * Error text, set if download fails.
      */
     error?: string | null;
+    /**
+     * Error type, may or may not be set in case of an error
+     */
+    error_type?: string | null;
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
-}
+    prefetched_info?: any;
+};

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { FileType } from './FileType';
 
-export interface DownloadArchiveRequest {
+export type DownloadArchiveRequest = {
-    sub: {
+    type?: FileType;
-archive_dir: string,
+    sub_id?: string;
-};
+};
-}

src/api-types/models/DownloadFileRequest.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface DownloadFileRequest {
+export type DownloadFileRequest = {
     uid?: string;
     uuid?: string;
     sub_id?: string;
     playlist_id?: string;
     url?: string;
     type?: FileType;
-}
+};

src/api-types/models/DownloadRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { CropFileSettings } from './CropFileSettings';
+import type { CropFileSettings } from './CropFileSettings';
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface DownloadRequest {
+export type DownloadRequest = {
     url: string;
     /**
      * Video format code. Overrides other quality options.
@@ -35,10 +35,18 @@ export interface DownloadRequest {
      * Height of the video, if known
      */
     selectedHeight?: string;
+    /**
+     * Max height that should be used, useful for playlists. selectedHeight will override this.
+     */
+    maxHeight?: string;
     /**
      * Specify ffmpeg/avconv audio quality
      */
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
-}
+    /**
+     * If using youtube-dl archive, download will ignore it
+     */
+    ignoreArchive?: boolean;
+};

src/api-types/models/DownloadResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Download } from './Download';
+import type { Download } from './Download';
 
-export interface DownloadResponse {
+export type DownloadResponse = {
     download?: Download;
-}
+};

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
-import { Subscription } from './Subscription';
+import type { Subscription } from './Subscription';
 
-export interface DownloadTwitchChatByVODIDRequest {
+export type DownloadTwitchChatByVODIDRequest = {
     /**
      * File ID
      */
@@ -20,4 +20,4 @@ export interface DownloadTwitchChatByVODIDRequest {
      */
     uuid?: string;
     sub?: Subscription;
-}
+};

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { TwitchChatMessage } from './TwitchChatMessage';
+import type { TwitchChatMessage } from './TwitchChatMessage';
 
-export interface DownloadTwitchChatByVODIDResponse {
+export type DownloadTwitchChatByVODIDResponse = {
     chat: Array<TwitchChatMessage>;
-}
+};

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DownloadVideosForSubscriptionRequest = {
-export interface DownloadVideosForSubscriptionRequest {
     subID: string;
-}
+};

src/api-types/models/FileType.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
 export enum FileType {
     AUDIO = 'audio',
     VIDEO = 'video',

src/api-types/models/FileTypeFilter.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum FileTypeFilter {
+    AUDIO_ONLY = 'audio_only',
+    VIDEO_ONLY = 'video_only',
+    BOTH = 'both',
+}

src/api-types/models/GenerateArgsResponse.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GenerateArgsResponse = {
-export interface GenerateArgsResponse {
     args?: Array<string>;
-}
+};

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GenerateNewApiKeyResponse = {
-export interface GenerateNewApiKeyResponse {
     new_api_key: string;
-}
+};

src/api-types/models/GetAllCategoriesResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Category } from './Category';
+import type { Category } from './Category';
 
-export interface GetAllCategoriesResponse {
+export type GetAllCategoriesResponse = {
     categories: Array<Category>;
-}
+};

src/api-types/models/GetAllDownloadsRequest.ts

@@ -2,10 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetAllDownloadsRequest = {
-export interface GetAllDownloadsRequest {
     /**
      * Filters downloads with the array
      */
     uids?: Array<string> | null;
-}
+};

src/api-types/models/GetAllDownloadsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Download } from './Download';
+import type { Download } from './Download';
 
-export interface GetAllDownloadsResponse {
+export type GetAllDownloadsResponse = {
     downloads?: Array<Download>;
-}
+};

src/api-types/models/GetAllFilesRequest.ts

@@ -0,0 +1,24 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileTypeFilter } from './FileTypeFilter';
+import type { Sort } from './Sort';
+
+export type GetAllFilesRequest = {
+    sort?: Sort;
+    range?: Array<number>;
+    /**
+     * Filter files by title
+     */
+    text_search?: string;
+    file_type_filter?: FileTypeFilter;
+    /**
+     * If set to true, only gets favorites
+     */
+    favorite_filter?: boolean;
+    /**
+     * Include if you want to filter by subscription
+     */
+    sub_id?: string;
+};

src/api-types/models/GetAllFilesResponse.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DatabaseFile } from './DatabaseFile';
+import type { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetAllFilesResponse {
+export type GetAllFilesResponse = {
     files: Array<DatabaseFile>;
     /**
      * All video playlists
      */
     playlists: Array<Playlist>;
-}
+};

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Subscription } from './Subscription';
+import type { Subscription } from './Subscription';
 
-export interface GetAllSubscriptionsResponse {
+export type GetAllSubscriptionsResponse = {
     subscriptions: Array<Subscription>;
-}
+};