Updated Angular to v14

.github/workflows/build.yml

@@ -15,9 +15,9 @@ jobs:
       - name: checkout code
         uses: actions/checkout@v2
       - name: setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '12'
           cache: 'npm'
       - name: install dependencies
         run: |
@@ -33,7 +33,7 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -55,7 +55,7 @@ jobs:
           Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
       - name: upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v1
         with:
           name: youtubedl-material
           path: build

.github/workflows/docker-pr.yml

@@ -18,21 +18,10 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
-      - name: setup platform emulator
+      - name: Build docker images
-        uses: docker/setup-qemu-action@v1
+        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr
-      - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v1
-      - name: build & push images
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
-          #platforms: linux/amd64
-          push: false
-          tags: tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker-release.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -60,10 +60,10 @@ jobs:
         uses: docker/setup-qemu-action@v1
 
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64/v8
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/docker.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -44,7 +44,7 @@ jobs:
         uses: docker/setup-qemu-action@v1
 
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Generate Docker image metadata
         id: docker-meta
@@ -63,7 +63,7 @@ jobs:
             type=sha,prefix=sha-,format=short
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}

.vscode/extensions.json

@@ -1,11 +0,0 @@
-{
-    "recommendations": [
-        "angular.ng-template",
-        "dbaeumer.vscode-eslint",
-        "waderyan.gitblame",
-        "42crunch.vscode-openapi",
-        "redhat.vscode-yaml",
-        "christian-kohler.npm-intellisense",
-        "hbenl.vscode-mocha-test-adapter"
-    ]
-}

.vscode/launch.json

@@ -4,20 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Dev: Debug Backend",
-            "request": "launch",
-            "runtimeArgs": [
-                "run-script",
-                "debug"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "node",
-            "cwd": "${workspaceFolder}/backend"
-        },
         {
             "type": "node",
             "request": "attach",

.vscode/settings.json

@@ -1,8 +0,0 @@
-{
-    "mochaExplorer.files": "backend/test/**/*.js",
-    "mochaExplorer.cwd": "backend",
-    "mochaExplorer.globImplementation": "vscode",
-    "mochaExplorer.env": {
-        "YTDL_MODE": "debug"
-    }
-}

.vscode/tasks.json

@@ -1,60 +1,25 @@
 {
 	"version": "2.0.0",
-	"windows": {
-        "options": {
-            "shell": {
-                "executable": "cmd.exe",
-                "args": [
-                    "/d", "/c"
-                ]
-            }
-        }
-    },
 	"tasks": [
 		{
 			"type": "npm",
 			"script": "start",
 			"problemMatcher": [],
 			"label": "Dev: start frontend",
-			"detail": "ng serve",
+			"detail": "ng serve"
-			"presentation": {
-				"echo": true,
-				"reveal": "always",
-				"focus": true,
-				"panel": "shared",
-				"showReuseMessage": true,
-				"clear": false
-			}
 		},
 		{
 			"label": "Dev: start backend",
 			"type": "shell",
-			"command": "node app.js",
+			"command": "set YTDL_MODE=debug && node app.js",
 			"options": {
-				"cwd": "./backend",
+				"cwd": "./backend"
-				"env": {
-					"YTDL_MODE": "debug"
-				}
 			},
 			"presentation": {
-				"echo": true,
 				"reveal": "always",
-				"focus": true,
+				"panel": "new"
-				"panel": "shared",
-				"showReuseMessage": true,
-				"clear": false
 			},
-			"problemMatcher": [],
+			"problemMatcher": []
-			"dependsOn": ["Dev: post-build"]
-		},
-		{
-			"label": "Dev: post-build",
-			"type": "shell",
-			"command": "node src/postbuild.mjs"
-		},
-		{
-			"label": "Dev: run all",
-			"dependsOn": ["Dev: start backend", "Dev: start frontend"]
 		}
 	]
 }

DEVELOPMENT.md

@@ -1,38 +0,0 @@
-<h1>Development</h1>
-
-- [First time...](#first-time)
-  - [Setup](#setup)
-  - [Startup](#startup)
-- [Debugging the backend (VSC)](#debugging-the-backend-vsc)
-- [Deploy changes](#deploy-changes)
-  - [Frontend](#frontend)
-  - [Backend](#backend)
-
-# First time...
-
-## Setup
-Checkout the repository and navigate to the `youtubedl-material` directory.
-```bash
-vim ./src/assets/default.json # Edit settings for your local environment. This config file is just the dev config file, if YTDL_MODE is not set to "debug", then ./backend/appdata/default.json will be used
-npm -g install pm2 # Install pm2
-npm install # Install dependencies for the frontend
-cd ./backend
-npm install # Install dependencies for the backend
-cd ..
-npm run build # Build the frontend
-```
-This step have to be done only once.
-
-## Startup
-Navigate to the `youtubedl-material/backend` directory and run `npm start`.
-
-# Debugging the backend (VSC)
-Open the `youtubedl-material` directory in Visual Studio Code and run the launch configuration `Dev: Debug Backend`.
-
-# Deploy changes
-
-## Frontend
-Navigate to the `youtubedl-material` directory and run `npm run build`. Restart the backend.
-
-## Backend
-Simply restart the backend.

Dockerfile

@@ -1,17 +1,14 @@
-# Fetching our utils
+# Fetching our ffmpeg
-FROM ubuntu:22.04 AS utils
+FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY docker-utils/*.sh .
+COPY ffmpeg-fetch.sh .
-RUN chmod +x *.sh
 RUN sh ./ffmpeg-fetch.sh
-RUN sh ./fetch-twitchdownloader.sh
 
 
-# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
+# Create our Ubuntu 22.04 with node 16
 # Go to 20.04
-FROM ubuntu:22.04 AS base
+FROM ubuntu:20.04 AS base
-ARG TARGETPLATFORM
 ARG DEBIAN_FRONTEND=noninteractive
 ENV UID=1000
 ENV GID=1000
@@ -19,30 +16,18 @@ ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
 ENV PM2_HOME=/app/pm2
 ENV ALLOW_CONFIG_MUTATIONS=true
-# Directy fetch specific version
-## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb
 RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
     apt update && \
-    apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt install -y --no-install-recommends nodejs && \
+    npm -g install npm && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
-    RUN case ${TARGETPLATFORM} in \
-         "linux/amd64")  NODE_ARCH=amd64   ;; \
-         "linux/arm")    NODE_ARCH=armhf  ;; \
-         "linux/arm/v7")    NODE_ARCH=armhf  ;; \
-         "linux/arm64")  NODE_ARCH=arm64  ;; \
-    esac \ 
-    && curl -L https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_$NODE_ARCH.deb -o ./nodejs.deb && \
-    apt update && \
-    apt install -y ./nodejs.deb && \
-    apt clean && \
-    rm -rf /var/lib/apt/lists/* &&\
-    rm nodejs.deb;
 
 
 # Build frontend
-ARG BUILDPLATFORM
+FROM base as frontend
-FROM --platform=${BUILDPLATFORM} node:16 as frontend
 RUN npm install -g @angular/cli
 WORKDIR /build
 COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
@@ -50,8 +35,6 @@ COPY [ "src/", "/build/src/" ]
 RUN npm install && \
     npm run build && \
     ls -al /build/backend/public
-RUN npm uninstall -g @angular/cli
-RUN rm -rf node_modules
 
 
 # Install backend deps
@@ -62,35 +45,21 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
-#FROM base as python
-# armv7 need build from source
-#WORKDIR /app
-#COPY docker-utils/GetTwitchDownloader.py .
-#RUN apt update && \
-#    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip python3-dev build-essential libffi-dev && \
-#    apt clean && \
-#    rm -rf /var/lib/apt/lists/*
-#RUN pip install PyGithub requests
-#RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
 RUN npm install -g pm2 && \
     apt update && \
-    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
-    pip install pycryptodomex && \
-    apt remove -y --purge build-essential && \
-    apt autoremove -y --purge && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
+RUN pip install tcd
 WORKDIR /app
 # User 1000 already exist from base image
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/usr/local/bin/TwitchDownloaderCLI"]
 COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-#COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"]
 RUN chmod +x /app/fix-scripts/*.sh
 # Add some persistence data
 #VOLUME ["/app/appdata"]

Public API v1.yaml

@@ -111,37 +111,6 @@ paths:
                 $ref: '#/components/schemas/GetAllFilesResponse'
       security:
         - Auth query parameter: []
-  /api/rss:
-    get:
-      tags:
-        - files
-      summary: Generates an RSS feed
-      description: Generates an RSS feed for downloaded files
-      operationId: get-rss
-      parameters:
-        - in: query
-          name: params
-          schema:
-            allOf:
-            - $ref: '#/components/schemas/GetAllFilesRequest'
-            - type: object
-              properties:
-                uuid:
-                  type: string
-                  description: user uid
-                  default: null
-          style: form
-          explode: true
-      responses:
-        '200':
-          description: OK
-          content:
-            text/plain:
-              schema:
-                type: string
-                description: RSS feed
-      security:
-        - Auth query parameter: []
   /api/getFile:
     post:
       tags:
@@ -578,69 +547,6 @@ paths:
           description: If the archive dir is not found, 404 is sent as a response
       security:
         - Auth query parameter: []
-  /api/deleteArchiveItems:
-    post:
-      tags:
-        - archive
-      summary: Delete item from archive
-      description: 'Deletes an item from the archive'
-      operationId: post-api-deleteArchiveItems
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/DeleteArchiveItemsRequest'
-      responses:
-        '200':
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/SuccessObject'
-      security:
-        - Auth query parameter: []
-  /api/importArchive:
-    post:
-      tags:
-        - archive
-      summary: Imports archive
-      description: 'Imports an existing archive.txt file'
-      operationId: post-api-importArchive
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/ImportArchiveRequest'
-      responses:
-        '200':
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/SuccessObject'
-      security:
-        - Auth query parameter: []
-  /api/uploadCookies:
-    post:
-      tags:
-        - downloader
-      summary: Upload cookies
-      description: 'Uploads cookies file to be used during downloading'
-      operationId: post-api-uploadCookies
-      requestBody:
-        content:
-          multipart/form-data:
-            schema:
-              $ref: '#/components/schemas/UploadCookiesRequest'
-      responses:
-        '200':
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/SuccessObject'
-      security:
-        - Auth query parameter: []
   /api/updaterStatus:
     get:
       tags:
@@ -905,7 +811,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/RestartDownloadResponse'
+                $ref: '#/components/schemas/SuccessObject'
       requestBody:
         content:
           application/json:
@@ -1683,10 +1589,6 @@ components:
           type: string
           description: Height of the video, if known
           example: '1080'
-        maxHeight:
-          type: string
-          description: Max height that should be used, useful for playlists. selectedHeight will override this.
-          example: '1080'
         maxBitrate:
           type: string
           description: Specify ffmpeg/avconv audio quality
@@ -1695,9 +1597,6 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
-        ignoreArchive:
-          type: boolean
-          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -1722,13 +1621,6 @@ components:
       properties:
         download:
           $ref: '#/components/schemas/Download'
-    RestartDownloadResponse:
-      allOf:
-        - $ref: '#/components/schemas/SuccessObject'
-        - type: object
-          properties:
-            new_download_uid:
-              type: string
     GetAllDownloadsRequest:
       type: object
       properties:
@@ -1758,14 +1650,14 @@ components:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
       required:
         - task_key
     UpdateTaskScheduleRequest:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         new_schedule:
           $ref: '#/components/schemas/Schedule'
       required:
@@ -1775,22 +1667,12 @@ components:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         new_data:
           type: object
       required:
         - task_key
         - new_data
-    UpdateTaskOptionsRequest:
-      type: object
-      properties:
-        task_key:
-          $ref: '#/components/schemas/TaskType'
-        new_options:
-          type: object
-      required:
-        - task_key
-        - new_options
     GetTaskResponse:
       type: object
       properties:
@@ -1859,39 +1741,29 @@ components:
             description: Two elements allowed, start index and end index
           minItems: 2
           maxItems: 2
-          default: null
         text_search:
           type: string
           description: Filter files by title
-          default: null
         file_type_filter:
           $ref: '#/components/schemas/FileTypeFilter'
-        favorite_filter:
-          type: boolean
-          description: If set to true, only gets favorites
-          default: false
         sub_id:
           type: string
           description: Include if you want to filter by subscription
-          default: null
     Sort:
       type: object
       properties:
         by:
           type: string
           description: Property to sort by
-          default: registered
         order:
           type: number
           description: 1 for ascending, -1 for descending
-          default: -1
     FileTypeFilter:
       type: string
       enum:
         - audio_only
         - video_only
         - both
-      default: both
     GetAllFilesResponse:
       required:
         - files
@@ -2009,11 +1881,16 @@ components:
           description: Number of files removed
     DeleteSubscriptionFileRequest:
       required:
-        - file_uid
+        - file
+        - sub
       type: object
       properties:
+        file:
+          type: string
         file_uid:
           type: string
+        sub:
+          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteForever:
           type: boolean
           description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -2153,83 +2030,17 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      type: object
-      properties:
-        type:
-          $ref: '#/components/schemas/FileType'
-        sub_id:
-          type: string
-    Archive:
       required:
-        - extractor
+        - sub
-        - id
-        - type
-        - title
-        - timestamp
-        - uid
       type: object
       properties:
-        extractor:
+        sub:
-          type: string
+          required:
-        id:
+            - archive_dir
-          type: string
+          type: object
-        type:
+          properties:
-          $ref: '#/components/schemas/FileType'
+            archive_dir:
-        title:
+              type: string
-          type: string
-        user_uid:
-          type: string
-        sub_id:
-          type: string
-        timestamp:
-          type: number
-        uid:
-          type: string
-    DeleteArchiveItemsRequest:
-      type: object
-      required:
-        - archives
-      properties:
-        archives:
-          type: array
-          items:
-            $ref: '#/components/schemas/Archive'
-    ImportArchiveRequest:
-      type: object
-      required:
-        - archive
-        - type
-      properties:
-        archive:
-          type: string
-        type:
-          $ref: '#/components/schemas/FileType'
-        sub_id:
-          type: string
-    GetArchivesRequest:
-      type: object
-      properties:
-        type:
-          $ref: '#/components/schemas/FileType'
-        sub_id:
-          type: string
-    GetArchivesResponse:
-      type: object
-      required:
-        - archives
-      properties:
-        archives:
-          type: array
-          items:
-            $ref: '#/components/schemas/Archive'
-    UploadCookiesRequest:
-      type: object
-      required:
-        - cookies
-      properties:
-        cookies:
-          type: string
-          format: binary
     UpdaterStatus:
       required:
         - details
@@ -2250,6 +2061,8 @@ components:
         tag:
           type: string
     DBInfoResponse:
+      required:
+        - db_info
       type: object
       properties:
         using_local_db:
@@ -2271,8 +2084,6 @@ components:
               $ref: '#/components/schemas/TableInfo'
             download_queue:
               $ref: '#/components/schemas/TableInfo'
-            archives:
-              $ref: '#/components/schemas/TableInfo'
     TransferDBResponse:
       required:
         - success
@@ -2572,7 +2383,6 @@ components:
         - upload_date
         - uploader
         - url
-        - favorite
       type: object
       properties:
         id:
@@ -2602,8 +2412,6 @@ components:
           type: string
         uid:
           type: string
-        user_uid:
-          type: string
         sharingEnabled:
           type: boolean
         category:
@@ -2622,8 +2430,6 @@ components:
         abr:
           type: number
           description: In Kbps
-        favorite:
-          type: boolean
     Playlist:
       required:
         - uids
@@ -2655,8 +2461,6 @@ components:
           type: string
         auto:
           type: boolean
-        sharingEnabled:
-          type: boolean
     Download:
       required:
         - url
@@ -2701,10 +2505,6 @@ components:
           type: string
           description: Error text, set if download fails.
           nullable: true
-        error_type:
-          type: string
-          description: Error type, may or may not be set in case of an error
-          nullable: true
         user_uid:
           type: string
         sub_id:
@@ -2726,7 +2526,7 @@ components:
       type: object
       properties:
         key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         title:
           type: string
         last_ran:
@@ -2742,20 +2542,7 @@ components:
         error:
           type: string
         schedule:
-          $ref: '#/components/schemas/Schedule'
-        options:
           type: object
-    TaskType:
-      type: string
-      enum:
-      - backup_local_db
-      - missing_files_check
-      - missing_db_records
-      - duplicate_files_check
-      - youtubedl_update_check
-      - delete_old_files
-      - import_legacy_archives
-      - rebuild_database
     Schedule:
       required:
         - type
@@ -2780,8 +2567,6 @@ components:
               type: number
             timestamp:
               type: number
-            tz:
-              type: string
     DBBackup:
       required:
         - name
@@ -2888,7 +2673,6 @@ components:
         - sharing
         - advanced_download
         - downloads_manager
-        - tasks_manager
     YesNo:
       type: string
       enum:
@@ -2971,44 +2755,6 @@ components:
           type: string
         date:
           type: string
-    Notification:
-      required:
-        - uid
-        - type
-        - text
-        - read
-        - timestamp
-      type: object
-      properties:
-        type:
-          $ref: '#/components/schemas/NotificationType'
-        uid:
-          type: string
-        user_uid:
-          type: string
-        action:
-          type: array
-          items:
-            $ref: '#/components/schemas/NotificationAction'
-        read:
-          type: boolean
-        data:
-          type: object
-        timestamp:
-          type: number
-    NotificationAction:
-      type: string
-      enum:
-      - play
-      - retry_download
-      - view_download_error
-      - view_tasks
-    NotificationType:
-      type: string
-      enum:
-      - download_complete
-      - download_error
-      - task_finished
     BaseChangePermissionsRequest:
       required:
         - permission
@@ -3140,29 +2886,6 @@ components:
                   type: array
                   items:
                     $ref: '#/components/schemas/UserPermission'
-    DeleteNotificationRequest:
-      required:
-        - uid
-      type: object
-      properties:
-        uid:
-          type: string
-    SetNotificationsToReadRequest:
-      required:
-        - uids
-      type: object
-      properties:
-        uids:
-          type: array
-          items:
-            type: string
-    GetNotificationsResponse:
-      type: object
-      properties:
-        notifications:
-          type: array
-          items:
-            $ref: '#/components/schemas/Notification'
   securitySchemes:
     Auth query parameter:
       name: apiKey

README.md

@@ -6,7 +6,7 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
@@ -14,7 +14,7 @@ Now with [Docker](#Docker) support!
 
 ## Getting Started
 
-Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
+Check out the prerequisites, and go to the installation section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
@@ -28,28 +28,13 @@ Dark mode:
 
 NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
 
-Required dependencies:
+Debian/Ubuntu:
-
-* Node.js 16
-* Python
-
-Optional dependencies:
-
-* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
-* [Twitch Downloader CLI](https://github.com/lay295/TwitchDownloader) (for downloading Twitch VOD chats)
-
-<details>
-  <summary>Debian/Ubuntu</summary>
 
 ```bash
-curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
 sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
 ```
 
-</details>
+CentOS 7:
-
-<details>
-  <summary>CentOS 7</summary>
 
 ```bash
 sudo yum install epel-release
@@ -57,16 +42,16 @@ sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfu
 sudo yum install centos-release-scl-rh
 sudo yum install rh-nodejs12
 scl enable rh-nodejs12 bash
-curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash -
 sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 ```
 
-</details>
+Optional dependencies:
+
+* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
-If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
-
 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.
@@ -85,9 +70,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
 
 To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
 
-Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm run build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
+Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
-
-Lastly, type `npm -g install pm2` to install pm2 globally.
 
 The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
 

angular.json

@@ -190,8 +190,5 @@
     "@schematics/angular:directive": {
       "prefix": "app"
     }
-  },
-  "cli": {
-    "analytics": false
   }
 }

backend/app.js

@@ -2,6 +2,7 @@ const { uuid } = require('uuidv4');
 const fs = require('fs-extra');
 const { promisify } = require('util');
 const auth_api = require('./authentication/auth');
+const winston = require('winston');
 const path = require('path');
 const compression = require('compression');
 const multer  = require('multer');
@@ -17,8 +18,6 @@ const URL = require('url').URL;
 const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
-const Feed = require('feed').Feed;
-const session = require('express-session');
 
 // needed if bin/details somehow gets deleted
 if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -33,8 +32,6 @@ const subscriptions_api = require('./subscriptions');
 const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
-const archive_api = require('./archive');
-const files_api = require('./files');
 
 var app = express();
 
@@ -72,8 +69,7 @@ db.defaults(
         downloads: {},
         subscriptions: [],
         files_to_db_migration_complete: false,
-        tasks_manager_role_migration_complete: false,
+        tasks_manager_role_migration_complete: false
-        archives_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -163,8 +159,6 @@ app.use(bodyParser.json());
 
 // use passport
 app.use(auth_api.passport.initialize());
-app.use(session({ secret: uuid(), resave: true, saveUninitialized: true }))
-app.use(auth_api.passport.session());
 
 // actual functions
 
@@ -175,10 +169,10 @@ async function checkMigrations() {
     if (!simplified_db_migration_complete) {
         logger.info('Beginning migration: 4.1->4.2+')
         let success = await simplifyDBFileStructure();
-        success = success && await files_api.addMetadataPropertyToDB('view_count');
+        success = success && await db_api.addMetadataPropertyToDB('view_count');
-        success = success && await files_api.addMetadataPropertyToDB('description');
+        success = success && await db_api.addMetadataPropertyToDB('description');
-        success = success && await files_api.addMetadataPropertyToDB('height');
+        success = success && await db_api.addMetadataPropertyToDB('height');
-        success = success && await files_api.addMetadataPropertyToDB('abr');
+        success = success && await db_api.addMetadataPropertyToDB('abr');
         // sets migration to complete
         db.set('simplified_db_migration_complete', true).write();
         if (success) { logger.info('4.1->4.2+ migration complete!'); }
@@ -205,15 +199,6 @@ async function checkMigrations() {
         db.set('tasks_manager_role_migration_complete', true).write();
     }
 
-    const archives_migration_complete = db.get('archives_migration_complete').value();
-    if (!archives_migration_complete) {
-        logger.info('Checking if archives have been migrated...');
-        const imported_archives = await archive_api.importArchives();
-        if (imported_archives) logger.info('Archives migration complete!');
-        else logger.error('Failed to migrate archives!');
-        db.set('archives_migration_complete', true).write();
-    }
-
     return true;
 }
 
@@ -525,6 +510,9 @@ async function loadConfig() {
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
+    // creates archive path if missing
+    await fs.ensureDir(utils.getArchiveFolder());
+
     // check migrations
     await checkMigrations();
 
@@ -535,7 +523,6 @@ async function loadConfig() {
     if (allowSubscriptions) {
         // set downloading to false
         let subscriptions = await subscriptions_api.getAllSubscriptions();
-        subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
         subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
         // runs initially, then runs every ${subscriptionCheckInterval} seconds
         const watchSubscriptionsInterval = function() {
@@ -571,7 +558,14 @@ function loadConfigValues() {
     url_domain = new URL(url);
 
     let logger_level = config_api.getConfigItem('ytdl_logger_level');
-    utils.updateLoggerLevel(logger_level);
+    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
+    if (!possible_levels.includes(logger_level)) {
+        logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
+        logger_level = 'info';
+    }
+    logger.level = logger_level;
+    winston.loggers.get('console').level = logger_level;
+    logger.transports[2].level = logger_level;
 }
 
 function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -706,7 +700,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -727,7 +721,7 @@ const optionalJwt = async function (req, res, next) {
         const uuid = using_body ? req.body.uuid : req.query.uuid;
         const uid = using_body ? req.body.uid : req.query.uid;
         const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
-        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
+        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
         if (file) {
             req.can_watch = true;
             return next();
@@ -779,7 +773,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
 
 app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
     const db_info = await db_api.getDBStats();
-    res.send(db_info);
+    res.send({db_info: db_info});
 });
 
 app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -819,13 +813,11 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
-        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings,
+        cropFileSettings: req.body.cropFileSettings
-        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -851,7 +843,6 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
         additionalArgs: req.body.additionalArgs,
         customOutput: req.body.customOutput,
         selectedHeight: req.body.selectedHeight,
-        maxHeight: req.body.maxHeight,
         customQualityConfiguration: req.body.customQualityConfiguration,
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
@@ -930,15 +921,35 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
+    let files = null;
     const sort = req.body.sort;
     const range = req.body.range;
     const text_search = req.body.text_search;
     const file_type_filter = req.body.file_type_filter;
-    const favorite_filter = req.body.favorite_filter;
     const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
+    const file_count = await db_api.getRecords('files', filter_obj, true);
+
+    files = JSON.parse(JSON.stringify(files));
 
     res.send({
         files: files,
@@ -1081,6 +1092,9 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
             await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
         } else if (is_playlist) {
             await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
+        } else if (type === 'subscription') {
+            // TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
+            //          time they are requested from the subscription directory.
         } else {
             // error
             success = false;
@@ -1095,7 +1109,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
     });
 });
 
-app.post('/api/incrementViewCount', async (req, res) => {
+app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
     let file_uid = req.body.file_uid;
     let sub_id = req.body.sub_id;
     let uuid = req.body.uuid;
@@ -1104,7 +1118,7 @@ app.post('/api/incrementViewCount', async (req, res) => {
         uuid = req.user.uid;
     }
 
-    const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
+    const file_obj = await db_api.getVideo(file_uid, uuid, sub_id);
 
     const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
     const new_view_count = current_view_count + 1;
@@ -1230,9 +1244,12 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
+    let file = req.body.file;
     let file_uid = req.body.file_uid;
+    let sub = req.body.sub;
+    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let success = await files_api.deleteFile(file_uid, deleteForever);
+    let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
 
     if (success) {
         res.send({
@@ -1320,7 +1337,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
 
-    const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1333,13 +1350,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
     let include_file_metadata = req.body.include_file_metadata;
 
-    const playlist = await files_api.getPlaylist(playlist_id, uuid);
+    const playlist = await db_api.getPlaylist(playlist_id, uuid);
     const file_objs = [];
 
     if (playlist && include_file_metadata) {
         for (let i = 0; i < playlist['uids'].length; i++) {
             const uid = playlist['uids'][i];
-            const file_obj = await files_api.getVideo(uid, uuid);
+            const file_obj = await db_api.getVideo(uid, uuid);
             if (file_obj) file_objs.push(file_obj);
             // TODO: remove file from playlist if could not be found
         }
@@ -1377,7 +1394,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
     playlist.uids.push(file_uid);
 
-    let success = await files_api.updatePlaylist(playlist);
+    let success = await db_api.updatePlaylist(playlist);
     res.send({
         success: success
     });
@@ -1385,7 +1402,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
 app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
     let playlist = req.body.playlist;
-    let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
+    let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
     res.send({
         success: success
     });
@@ -1413,9 +1430,10 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
 app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const uid = req.body.uid;
     const blacklistMode = req.body.blacklistMode;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     let wasDeleted = false;
-    wasDeleted = await files_api.deleteFile(uid, blacklistMode);
+    wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1447,7 +1465,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode);
+        wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1473,10 +1491,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     if (playlist_id) {
         zip_file_generated = true;
         const playlist_files_to_download = [];
-        const playlist = await files_api.getPlaylist(playlist_id, uuid);
+        const playlist = await db_api.getPlaylist(playlist_id, uuid);
         for (let i = 0; i < playlist['uids'].length; i++) {
             const playlist_file_uid = playlist['uids'][i];
-            const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
+            const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
             playlist_files_to_download.push(file_obj);
         }
 
@@ -1490,7 +1508,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
         // generate zip
         file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
     } else {
-        const file_obj = await files_api.getVideo(uid, uuid, sub_id)
+        const file_obj = await db_api.getVideo(uid, uuid, sub_id)
         file_path_to_download = file_obj.path;
     }
     if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
@@ -1508,69 +1526,20 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     });
 });
 
-app.post('/api/getArchives', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-    const sub_id = req.body.sub_id;
-    const filter_obj = {user_uid: uuid, sub_id: sub_id};
-    const type = req.body.type;
-
-    // we do this for file types because if type is null, that means get files of all types
-    if (type) filter_obj['type'] = type;
-
-    const archives = await db_api.getRecords('archives', filter_obj);
-
-    res.send({
-        archives: archives
-    });
-});
-
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
+    let sub = req.body.sub;
-    const sub_id = req.body.sub_id; 
+    let archive_dir = sub.archive;
-    const type = req.body.type;
 
-    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
+    let full_archive_path = path.join(archive_dir, 'archive.txt');
 
-    if (archive_text !== null && archive_text !== undefined) {
+    if (await fs.pathExists(full_archive_path)) {
-        res.setHeader('Content-type', "application/octet-stream");
+        res.sendFile(full_archive_path);
-        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
-        res.send(archive_text);
     } else {
-        res.sendStatus(400);
+        res.sendStatus(404);
     }
 
 });
 
-app.post('/api/importArchive', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-    const archive = req.body.archive;
-    const sub_id = req.body.sub_id; 
-    const type = req.body.type;
-
-    const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
-
-    const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
-
-    res.send({
-        success: !!imported_count,
-        imported_count: imported_count
-    });
-});
-
-app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-    const archives = req.body.archives;
-
-    let success = true;
-    for (const archive of archives) {
-        success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
-    }
-
-    res.send({
-        success: success
-    });
-});
-
 var upload_multer = multer({ dest: __dirname + '/appdata/' });
 app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
     const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1637,12 +1606,12 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
 
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
-        file_obj = await files_api.getVideo(uid, uuid, sub_id);
+        file_obj = await db_api.getVideo(uid, uuid, sub_id);
         if (file_obj) file_path = file_obj['path'];
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
-        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
+        logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1762,8 +1731,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
 
 app.post('/api/restartDownload', optionalJwt, async (req, res) => {
     const download_uid = req.body.download_uid;
-    const new_download = await downloader_api.restartDownload(download_uid);
+    const success = await downloader_api.restartDownload(download_uid);
-    res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
+    res.send({success: success});
 });
 
 app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1840,15 +1809,6 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
-app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
-    const task_key = req.body.task_key;
-    const new_options = req.body.new_options;
-  
-    const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
-
-    res.send({success: success});
-});
-
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
     fs.ensureDirSync(backup_dir);
@@ -1929,38 +1889,9 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
 
 // user authentication
 
-app.post('/api/auth/register', optionalJwt, async (req, res) => {
+app.post('/api/auth/register'
-    const userid = req.body.userid;
+        , optionalJwt
-    const username = req.body.username;
+        , auth_api.registerUser);
-    const plaintextPassword = req.body.password;
-
-    if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
-        logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
-        res.sendStatus(409);
-        return;
-    }
-
-    if (plaintextPassword === "") {
-        logger.error(`Registration failed for user ${userid}. A password must be provided.`);
-        res.sendStatus(409);
-        return;
-    }
-
-    if (!userid || !username) {
-        logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
-    }
-  
-    const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
-  
-    if (!new_user) {
-      res.sendStatus(409);
-      return;
-    }
-  
-    res.send({
-      user: new_user
-    });
-});
 app.post('/api/auth/login'
         , auth_api.passport.authenticate(['local', 'ldapauth'], {})
         , auth_api.generateJWT
@@ -2012,7 +1943,18 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
 app.post('/api/deleteUser', optionalJwt, async (req, res) => {
     let uid = req.body.uid;
     try {
-        const success = await auth_api.deleteUser(uid);
+        let success = false;
+        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        const user_folder = path.join(__dirname, usersFileFolder, uid);
+        const user_db_obj = await db_api.getRecord('users', {uid: uid});
+        if (user_db_obj) {
+            // user exists, let's delete
+            await fs.remove(user_folder);
+            await db_api.removeRecord('users', {uid: uid});
+            success = true;
+        } else {
+            logger.error(`Could not find user with uid ${uid}`);
+        }
         res.send({success: success});
     } catch (err) {
         logger.error(err);
@@ -2050,93 +1992,6 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
-// notifications
-
-app.post('/api/getNotifications', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-
-    const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
-
-    res.send({notifications: notifications});
-});
-
-// set notifications to read
-app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-
-    const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
-
-    res.send({success: success});
-});
-
-app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
-    const uid = req.isAuthenticated() ? req.user.uid : null;
-
-    const success = await db_api.removeRecord('notifications', {uid: uid});
-
-    res.send({success: success});
-});
-
-app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
-    const uuid = req.isAuthenticated() ? req.user.uid : null;
-
-    const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
-
-    res.send({success: success});
-});
-
-// rss feed
-
-app.get('/api/rss', async function (req, res) {
-    if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
-        logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
-        res.sendStatus(403);
-        return;
-    }
-
-    // these are returned
-    const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
-    const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
-    const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
-    const file_type_filter = req.query.file_type_filter;
-    const favorite_filter = req.query.favorite_filter === 'true';
-    const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
-    const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
-
-    const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
-
-    const feed = new Feed({
-            title: 'Downloads',
-            description: 'YoutubeDL-Material downloads',
-            id: utils.getBaseURL(),
-            link: utils.getBaseURL(),
-            image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
-            favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
-            generator: 'YoutubeDL-Material'
-    });
-
-    files.forEach(file => {
-        feed.addItem({
-            title: file.title,
-            link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
-            description: file.description,
-            author: [
-                {
-                    name: file.uploader,
-                    link: file.url
-                }
-            ],
-            contributor: [],
-            date: file.timestamp,
-            // https://stackoverflow.com/a/45415677/8088021
-            image: file.thumbnailURL.replace('&', '&amp;')
-        });
-      });
-    res.send(feed.rss2());
-});
-
-// web server
-
 app.use(function(req, res, next) {
     //if the request is not html then move along
     var accept = req.accepts('html', 'json', 'xml');

backend/appdata/default.json

@@ -23,12 +23,7 @@
       "download_only_mode": false,
       "allow_autoplay": true,
       "enable_downloads_manager": true,
-      "allow_playlist_categorization": true,
+      "allow_playlist_categorization": true
-      "force_autoplay": false,
-      "enable_notifications": true,
-      "enable_all_notifications": true,
-      "allowed_notification_types": [],
-      "enable_rss_feed": false
     },
     "API": {
       "use_API_key": false,
@@ -40,18 +35,7 @@
       "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
-      "generate_NFO_files": false,
+      "generate_NFO_files": false
-      "use_ntfy_API": false,
-      "ntfy_topic_URL": "",
-      "use_gotify_API": false,
-      "gotify_server_URL": "",
-      "gotify_app_token": "",
-      "use_telegram_API": false,
-      "telegram_bot_token": "",
-      "telegram_chat_id": "",
-      "webhook_URL": "",
-      "discord_webhook_URL": "",
-      "slack_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",

backend/archive.js

@@ -1,91 +0,0 @@
-const path = require('path');
-const fs = require('fs-extra');
-const { uuid } = require('uuidv4');
-
-const db_api = require('./db');
-
-exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
-    const filter = {user_uid: user_uid, sub_id: sub_id};
-    if (type) filter['type'] = type;
-    const archive_items = await db_api.getRecords('archives', filter);
-    const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
-    return archive_item_lines.join('\n');
-}
-
-exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
-    const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
-    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
-    return success;
-}
-
-exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
-    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
-    return success;
-}
-
-exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
-    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
-    return !!archive_item;
-}
-
-exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
-    let archive_import_count = 0;
-    const lines = archive_text.split('\n');
-    for (let line of lines) {
-        const archive_line_parts = line.trim().split(' ');
-        // should just be the extractor and the video ID
-        if (archive_line_parts.length !== 2) {
-            continue;
-        }
-
-        const extractor = archive_line_parts[0];
-        const id        = archive_line_parts[1];
-        if (!extractor || !id) continue;
-
-        // we can't do a bulk write because we need to avoid duplicate archive items existing in db
-
-        const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
-        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
-        archive_import_count++;
-    }
-    return archive_import_count;
-}
-
-exports.importArchives = async () => {
-    const imported_archives = [];
-    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
-
-    // run through check list and check each file to see if it's missing from the db
-    for (let i = 0; i < dirs_to_check.length; i++) {
-        const dir_to_check = dirs_to_check[i];
-        if (!dir_to_check['archive_path']) continue;
-
-        const files_to_import = [
-            path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
-            path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
-        ]
-
-        for (const file_to_import of files_to_import) {
-            const file_exists = await fs.pathExists(file_to_import);
-            if (!file_exists) continue;
-
-            const archive_text = await fs.readFile(file_to_import, 'utf8');
-            await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
-            imported_archives.push(file_to_import);
-        }
-    }
-    return imported_archives;
-}
-
-const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
-    return {
-        extractor: extractor,
-        id: id,
-        type: type,
-        title: title,
-        user_uid: user_uid ? user_uid : null,
-        sub_id: sub_id ? sub_id : null,
-        timestamp: Date.now() / 1000,
-        uid: uuid()
-    }
-}

backend/authentication/auth.js

@@ -1,13 +1,11 @@
 const config_api = require('../config');
-const CONSTS = require('../consts');
+const consts = require('../consts');
 const logger = require('../logger');
 const db_api = require('../db');
 
 const jwt = require('jsonwebtoken');
 const { uuid } = require('uuidv4');
 const bcrypt = require('bcryptjs');
-const fs = require('fs-extra');
-const path = require('path');
 
 var LocalStrategy = require('passport-local').Strategy;
 var LdapStrategy = require('passport-ldapauth');
@@ -18,7 +16,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
 let SERVER_SECRET = null;
 let JWT_EXPIRATION = null;
 let opts = null;
-let saltRounds = 10;
+let saltRounds = null;
 
 exports.initialize = function () {
   /*************************
@@ -33,14 +31,9 @@ exports.initialize = function () {
       });
   }
 
-  // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
+  saltRounds = 10;
+
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
-  if (!(+JWT_EXPIRATION)) {
-    logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
-    JWT_EXPIRATION = 86400;
-  } else {
-    JWT_EXPIRATION = +JWT_EXPIRATION;
-  }
 
   SERVER_SECRET = null;
   if (db_api.users_db.get('jwt_secret').value()) {
@@ -68,7 +61,14 @@ exports.initialize = function () {
 const setupRoles = async () => {
   const required_roles = {
     admin: {
-        permissions: CONSTS.AVAILABLE_PERMISSIONS
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
     },
     user: {
         permissions: [
@@ -106,41 +106,55 @@ exports.passport.deserializeUser(function(user, done) {
 /***************************************
  * Register user with hashed password
  **************************************/
+exports.registerUser = async function(req, res) {
+  var userid = req.body.userid;
+  var username = req.body.username;
+  var plaintextPassword = req.body.password;
 
-exports.registerUser = async (userid, username, plaintextPassword) => {
+  if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
-  const hash = await bcrypt.hash(plaintextPassword, saltRounds);
+    res.sendStatus(409);
-  const new_user = generateUserObject(userid, username, hash);
+    logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
-  // check if user exists
+    return;
-  if (await db_api.getRecord('users', {uid: userid})) {
-    // user id is taken!
-    logger.error('Registration failed: UID is already taken!');
-    return null;
-  } else if (await db_api.getRecord('users', {name: username})) {
-      // user name is taken!
-      logger.error('Registration failed: User name is already taken!');
-      return null;
-  } else {
-    // add to db
-    await db_api.insertRecordIntoTable('users', new_user);
-    logger.verbose(`New user created: ${new_user.name}`);
-    return new_user;
   }
-}
 
-exports.deleteUser = async (uid) => {
+  if (plaintextPassword === "") {
-  let success = false;
+    res.sendStatus(400);
-  let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    logger.error(`Registration failed for user ${userid}. A password must be provided.`);
-  const user_folder = path.join(__dirname, usersFileFolder, uid);
+    return;
-  const user_db_obj = await db_api.getRecord('users', {uid: uid});
-  if (user_db_obj) {
-      // user exists, let's delete
-      await fs.remove(user_folder);
-      await db_api.removeRecord('users', {uid: uid});
-      success = true;
-  } else {
-      logger.error(`Could not find user with uid ${uid}`);
   }
-  return success;
+
+  bcrypt.hash(plaintextPassword, saltRounds)
+    .then(async function(hash) {
+      let new_user = generateUserObject(userid, username, hash);
+      // check if user exists
+      if (await db_api.getRecord('users', {uid: userid})) {
+        // user id is taken!
+        logger.error('Registration failed: UID is already taken!');
+        res.status(409).send('UID is already taken!');
+      } else if (await db_api.getRecord('users', {name: username})) {
+          // user name is taken!
+          logger.error('Registration failed: User name is already taken!');
+          res.status(409).send('User name is already taken!');
+      } else {
+        // add to db
+        await db_api.insertRecordIntoTable('users', new_user);
+        logger.verbose(`New user created: ${new_user.name}`);
+        res.send({
+          user: new_user
+        });
+      }
+    })
+    .then(function(result) {
+
+    })
+    .catch(function(err) {
+      logger.error(err);
+      if( err.code == 'ER_DUP_ENTRY' ) {
+        res.status(409).send('UserId already taken');
+      } else {
+        res.sendStatus(409);
+      }
+    });
 }
 
 /***************************************
@@ -221,7 +235,7 @@ exports.returnAuthResponse = async function(req, res) {
     user: req.user,
     token: req.token,
     permissions: await exports.userPermissions(req.user.uid),
-    available_permissions: CONSTS.AVAILABLE_PERMISSIONS
+    available_permissions: consts['AVAILABLE_PERMISSIONS']
   });
 }
 
@@ -305,7 +319,7 @@ exports.getUserVideos = async function(user_uid, type) {
 }
 
 exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
-  let file = await db_api.getRecord('files', {uid: file_uid});
+  let file = await db_api.getRecord('files', {file_uid: file_uid});
 
   // prevent unauthorized users from accessing the file info
   if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -392,8 +406,8 @@ exports.userPermissions = async function(user_uid) {
   const role_obj = await db_api.getRecord('roles', {key: role});
   const role_permissions = role_obj['permissions'];
 
-  for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
+  for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
-    let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
+    let permission = consts['AVAILABLE_PERMISSIONS'][i];
 
     const user_has_explicit_permission = user_obj['permissions'].includes(permission);
     const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

backend/config.js

@@ -185,6 +185,7 @@ const DEFAULT_CONFIG = {
         "default_file_output": "",
         "use_youtubedl_archive": false,
         "custom_args": "",
+        "safe_download_override": false,
         "include_thumbnail": true,
         "include_metadata": true,
         "max_concurrent_downloads": 5,
@@ -195,33 +196,21 @@ const DEFAULT_CONFIG = {
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "force_autoplay": false,
+        "allow_autoplay": true,
         "enable_downloads_manager": true,
-        "allow_playlist_categorization": true,
+        "allow_playlist_categorization": true
-        "enable_notifications": true,
-        "enable_all_notifications": true,
-        "allowed_notification_types": [],
-        "enable_rss_feed": false,
       },
       "API": {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
+        "use_twitch_API": false,
+        "twitch_client_ID": "",
+        "twitch_client_secret": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
-        "generate_NFO_files": false,
+        "generate_NFO_files": false
-        "use_ntfy_API": false,
-        "ntfy_topic_URL": "",
-        "use_gotify_API": false,
-        "gotify_server_URL": "",
-        "gotify_app_token": "",
-        "use_telegram_API": false,
-        "telegram_bot_token": "",
-        "telegram_chat_id": "",
-        "webhook_URL": "",
-        "discord_webhook_URL": "",
-        "slack_webhook_URL": "",
       },
       "Themes": {
         "default_theme": "default",

backend/consts.js

@@ -30,6 +30,10 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'
     },
+    'ytdl_safe_download_override': {
+        'key': 'ytdl_safe_download_override',
+        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
+    },
     'ytdl_include_thumbnail': {
         'key': 'ytdl_include_thumbnail',
         'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -64,9 +68,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_force_autoplay': {
+    'ytdl_allow_autoplay': {
-        'key': 'ytdl_force_autoplay',
+        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.force_autoplay'
+        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -76,22 +80,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_allow_playlist_categorization',
         'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
-    'ytdl_enable_notifications': {
-        'key': 'ytdl_enable_notifications',
-        'path': 'YoutubeDLMaterial.Extra.enable_notifications'
-    },
-    'ytdl_enable_all_notifications': {
-        'key': 'ytdl_enable_all_notifications',
-        'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
-    },
-    'ytdl_allowed_notification_types': {
-        'key': 'ytdl_allowed_notification_types',
-        'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
-    },
-    'ytdl_enable_rss_feed': {
-        'key': 'ytdl_enable_rss_feed',
-        'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
-    },
 
     // API
     'ytdl_use_api_key': {
@@ -110,6 +98,18 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
+    'ytdl_use_twitch_api': {
+        'key': 'ytdl_use_twitch_api',
+        'path': 'YoutubeDLMaterial.API.use_twitch_API'
+    },
+    'ytdl_twitch_client_id': {
+        'key': 'ytdl_twitch_client_id',
+        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
+    },
+    'ytdl_twitch_client_secret': {
+        'key': 'ytdl_twitch_client_secret',
+        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
+    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -122,50 +122,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_generate_nfo_files',
         'path': 'YoutubeDLMaterial.API.generate_NFO_files'
     },
-    'ytdl_use_ntfy_API': {
-        'key': 'ytdl_use_ntfy_API',
-        'path': 'YoutubeDLMaterial.API.use_ntfy_API'
-    },
-    'ytdl_ntfy_topic_url': {
-        'key': 'ytdl_ntfy_topic_url',
-        'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
-    },
-    'ytdl_use_gotify_API': {
-        'key': 'ytdl_use_gotify_API',
-        'path': 'YoutubeDLMaterial.API.use_gotify_API'
-    },
-    'ytdl_gotify_server_url': {
-        'key': 'ytdl_gotify_server_url',
-        'path': 'YoutubeDLMaterial.API.gotify_server_URL'
-    },
-    'ytdl_gotify_app_token': {
-        'key': 'ytdl_gotify_app_token',
-        'path': 'YoutubeDLMaterial.API.gotify_app_token'
-    },
-    'ytdl_use_telegram_API': {
-        'key': 'ytdl_use_telegram_API',
-        'path': 'YoutubeDLMaterial.API.use_telegram_API'
-    },
-    'ytdl_telegram_bot_token': {
-        'key': 'ytdl_telegram_bot_token',
-        'path': 'YoutubeDLMaterial.API.telegram_bot_token'
-    },
-    'ytdl_telegram_chat_id': {
-        'key': 'ytdl_telegram_chat_id',
-        'path': 'YoutubeDLMaterial.API.telegram_chat_id'
-    },
-    'ytdl_webhook_url': {
-        'key': 'ytdl_webhook_url',
-        'path': 'YoutubeDLMaterial.API.webhook_URL'
-    },
-    'ytdl_discord_webhook_url': {
-        'key': 'ytdl_discord_webhook_url',
-        'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
-    },
-    'ytdl_slack_webhook_url': {
-        'key': 'ytdl_slack_webhook_url',
-        'path': 'YoutubeDLMaterial.API.slack_webhook_URL'
-    },
 
 
     // Themes
@@ -347,11 +303,7 @@ const YTDL_ARGS_WITH_VALUES = [
     '--convert-subs'
 ];
 
-exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
-
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
-exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
+exports.CURRENT_VERSION = 'v4.3';
-
-exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -1,11 +1,10 @@
-const fs = require('fs-extra')
+var fs = require('fs-extra')
-const path = require('path')
+var path = require('path')
 const { MongoClient } = require("mongodb");
 const { uuid } = require('uuidv4');
-const _ = require('lodash');
 
 const config_api = require('./config');
-const utils = require('./utils')
+var utils = require('./utils')
 const logger = require('./logger');
 
 const low = require('lowdb')
@@ -59,13 +58,6 @@ const tables = {
         name: 'tasks',
         primary_key: 'key'
     },
-    notifications: {
-        name: 'notifications',
-        primary_key: 'uid'
-    },
-    archives: {
-        name: 'archives'
-    },
     test: {
         name: 'test'
     }
@@ -156,7 +148,6 @@ exports._connectToDB = async (custom_connection_string = null) => {
                 await database.collection(table).createIndex(text_search);
             }
         });
-        using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
         return true;
     } catch(err) {
         logger.error(err);
@@ -167,9 +158,82 @@ exports._connectToDB = async (custom_connection_string = null) => {
     }
 }
 
-exports.setVideoProperty = async (file_uid, assignment_obj) => {
+exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
-    // TODO: check if video exists, throw error if not
+    if (!file_object) file_object = generateFileObject(file_path, type);
-    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
+    if (!file_object) {
+        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
+        return false;
+    }
+
+    utils.fixVideoMetadataPerms(file_path, type);
+
+    // add thumbnail path
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
+
+    // if category exists, only include essential info
+    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
+
+    // modify duration
+    if (cropFileSettings) {
+        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
+    }
+
+    if (user_uid) file_object['user_uid'] = user_uid;
+    if (sub_id) file_object['sub_id'] = sub_id;
+
+    const file_obj = await registerFileDBManual(file_object);
+
+    // remove metadata JSON if needed
+    if (!config_api.getConfigItem('ytdl_include_metadata')) {
+        utils.deleteJSONFile(file_path, type)
+    }
+
+    return file_obj;
+}
+
+async function registerFileDBManual(file_object) {
+    // add additional info
+    file_object['uid'] = uuid();
+    file_object['registered'] = Date.now();
+    path_object = path.parse(file_object['path']);
+    file_object['path'] = path.format(path_object);
+
+    await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+
+    return file_object;
+}
+
+function generateFileObject(file_path, type) {
+    var jsonobj = utils.getJSON(file_path, type);
+    if (!jsonobj) {
+        return null;
+    } else if (!jsonobj['_filename']) {
+        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
+        return null;
+    }
+    const ext = (type === 'audio') ? '.mp3' : '.mp4'
+    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
+    // console.
+    var stats = fs.statSync(true_file_path);
+
+    const file_id = utils.removeFileExtension(path.basename(file_path));
+    var title = jsonobj.title;
+    var url = jsonobj.webpage_url;
+    var uploader = jsonobj.uploader;
+    var upload_date = utils.formatDateString(jsonobj.upload_date);
+
+    var size = stats.size;
+
+    var thumbnail = jsonobj.thumbnail;
+    var duration = jsonobj.duration;
+    var isaudio = type === 'audio';
+    var description = jsonobj.description;
+    var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
+    return file_obj;
+}
+
+function getAppendedBasePathSub(sub, base_path) {
+    return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
 exports.getFileDirectoriesAndDBs = async () => {
@@ -188,16 +252,13 @@ exports.getFileDirectoriesAndDBs = async () => {
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'audio'),
                 user_uid: user.uid,
-                type: 'audio',
+                type: 'audio'
-                archive_path: utils.getArchiveFolder('audio', user.uid)
             });
 
             // add user's video dir to check list
             dirs_to_check.push({
                 basePath: path.join(usersFileFolder, user.uid, 'video'),
-                user_uid: user.uid,
+                type: 'video'
-                type: 'video',
-                archive_path: utils.getArchiveFolder('video', user.uid)
             });
         }
     } else {
@@ -207,15 +268,13 @@ exports.getFileDirectoriesAndDBs = async () => {
         // add audio dir to check list
         dirs_to_check.push({
             basePath: audioFolderPath,
-            type: 'audio',
+            type: 'audio'
-            archive_path: utils.getArchiveFolder('audio')
         });
 
         // add video dir to check list
         dirs_to_check.push({
             basePath: videoFolderPath,
-            type: 'video',
+            type: 'video'
-            archive_path: utils.getArchiveFolder('video')
         });
     }
 
@@ -236,14 +295,254 @@ exports.getFileDirectoriesAndDBs = async () => {
                                       : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
             user_uid: subscription_to_check.user_uid,
             type: subscription_to_check.type,
-            sub_id: subscription_to_check['id'],
+            sub_id: subscription_to_check['id']
-            archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
         });
     }
 
     return dirs_to_check;
 }
 
+exports.importUnregisteredFiles = async () => {
+    const imported_files = [];
+    const dirs_to_check = await exports.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        // recursively get all files in dir's path
+        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
+
+        for (let j = 0; j < files.length; j++) {
+            const file = files[j];
+
+            // check if file exists in db, if not add it
+            const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
+            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
+            if (!file_is_registered) {
+                // add additional info
+                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
+                if (file_obj) {
+                    imported_files.push(file_obj['uid']);
+                    logger.verbose(`Added discovered file to the database: ${file.id}`);
+                } else {
+                    logger.error(`Failed to import ${file['path']} automatically.`);
+                }
+            }
+        }
+    }
+    return imported_files;
+}
+
+exports.addMetadataPropertyToDB = async (property_key) => {
+    try {
+        const dirs_to_check = await exports.getFileDirectoriesAndDBs();
+        const update_obj = {};
+        for (let i = 0; i < dirs_to_check.length; i++) {
+            const dir_to_check = dirs_to_check[i];
+
+            // recursively get all files in dir's path
+            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (file[property_key]) {
+                    update_obj[file.uid] = {[property_key]: file[property_key]};
+                }
+            }
+        }
+
+        return await exports.bulkUpdateRecords('files', 'uid', update_obj);
+    } catch(err) {
+        logger.error(err);
+        return false;
+    }
+}
+
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
+    const first_video = await exports.getVideo(uids[0]);
+    const thumbnailToUse = first_video['thumbnailURL'];
+    
+    let new_playlist = {
+        name: playlist_name,
+        uids: uids,
+        id: uuid(),
+        thumbnailURL: thumbnailToUse,
+        registered: Date.now(),
+        randomize_order: false
+    };
+
+    new_playlist.user_uid = user_uid ? user_uid : undefined;
+
+    await exports.insertRecordIntoTable('playlists', new_playlist);
+    
+    const duration = await exports.calculatePlaylistDuration(new_playlist);
+    await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
+
+    return new_playlist;
+}
+
+exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
+    let playlist = await exports.getRecord('playlists', {id: playlist_id});
+
+    if (!playlist) {
+        playlist = await exports.getRecord('categories', {uid: playlist_id});
+        if (playlist) {
+            const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
+            playlist['uids'] = uids;
+            playlist['auto'] = true;
+        }
+    }
+
+    // converts playlists to new UID-based schema
+    if (playlist && playlist['fileNames'] && !playlist['uids']) {
+        playlist['uids'] = [];
+        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
+        for (let i = 0; i < playlist['fileNames'].length; i++) {
+            const fileName = playlist['fileNames'][i];
+            const uid = await exports.getVideoUIDByID(fileName, user_uid);
+            if (uid) playlist['uids'].push(uid);
+            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
+        }
+        exports.updatePlaylist(playlist, user_uid);
+    }
+
+    // prevent unauthorized users from accessing the file info
+    if (require_sharing && !playlist['sharingEnabled']) return null;
+
+    return playlist;
+}
+
+exports.updatePlaylist = async (playlist) => {
+    let playlistID = playlist.id;
+
+    const duration = await exports.calculatePlaylistDuration(playlist);
+    playlist.duration = duration;
+
+    return await exports.updateRecord('playlists', {id: playlistID}, playlist);
+}
+
+exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
+    let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
+
+    if (!success) {
+        success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
+    }
+
+    if (!success) {
+        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
+    }
+
+    return success;
+}
+
+exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
+    if (!playlist_file_objs) {
+        playlist_file_objs = [];
+        for (let i = 0; i < playlist['uids'].length; i++) {
+            const uid = playlist['uids'][i];
+            const file_obj = await exports.getVideo(uid);
+            if (file_obj) playlist_file_objs.push(file_obj);
+        }
+    }
+
+    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
+}
+
+exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid, uuid);
+    const type = file_obj.isAudio ? 'audio' : 'video';
+    const folderPath = path.dirname(file_obj.path);
+    const ext = type === 'audio' ? 'mp3' : 'mp4';
+    const name = file_obj.id;
+    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
+
+    var jsonPath = `${file_obj.path}.info.json`;
+    var altJSONPath = `${filePathNoExtension}.info.json`;
+    var thumbnailPath = `${filePathNoExtension}.webp`;
+    var altThumbnailPath = `${filePathNoExtension}.jpg`;
+
+    jsonPath = path.join(__dirname, jsonPath);
+    altJSONPath = path.join(__dirname, altJSONPath);
+
+    let jsonExists = await fs.pathExists(jsonPath);
+    let thumbnailExists = await fs.pathExists(thumbnailPath);
+
+    if (!jsonExists) {
+        if (await fs.pathExists(altJSONPath)) {
+            jsonExists = true;
+            jsonPath = altJSONPath;
+        }
+    }
+
+    if (!thumbnailExists) {
+        if (await fs.pathExists(altThumbnailPath)) {
+            thumbnailExists = true;
+            thumbnailPath = altThumbnailPath;
+        }
+    }
+
+    let fileExists = await fs.pathExists(file_obj.path);
+
+    if (config_api.descriptors[uid]) {
+        try {
+            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
+                config_api.descriptors[uid][i].destroy();
+            }
+        } catch(e) {
+
+        }
+    }
+
+    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_path = utils.getArchiveFolder(type, uuid);
+
+        // get ID from JSON
+
+        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let id = null;
+        if (jsonobj) id = jsonobj.id;
+
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
+    }
+
+    if (jsonExists) await fs.unlink(jsonPath);
+    if (thumbnailExists) await fs.unlink(thumbnailPath);
+
+    await exports.removeRecord('files', {uid: uid});
+
+    if (fileExists) {
+        await fs.unlink(file_obj.path);
+        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
+            return false;
+        } else {
+            return true;
+        }
+    } else {
+        // TODO: tell user that the file didn't exist
+        return true;
+    }
+}
+
+// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
+exports.getVideoUIDByID = async (file_id, uuid = null) => {
+    const file_obj = await exports.getRecord('files', {id: file_id});
+    return file_obj ? file_obj['uid'] : null;
+}
+
+exports.getVideo = async (file_uid) => {
+    return await exports.getRecord('files', {uid: file_uid});
+}
+
+exports.getFiles = async (uuid = null) => {
+    return await exports.getRecords('files', {user_uid: uuid});
+}
+
+exports.setVideoProperty = async (file_uid, assignment_obj) => {
+    // TODO: check if video exists, throw error if not
+    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
+}
+
 // Basic DB functions
 
 // Create
@@ -251,7 +550,7 @@ exports.getFileDirectoriesAndDBs = async () => {
 exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
     // local db override
     if (using_local_db) {
-        if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
+        if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
         local_db.get(table).push(doc).write();
         return true;
     }
@@ -354,15 +653,9 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 
 // Update
 
-exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
+exports.updateRecord = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        if (nested_mode) {
-            // if object is nested we need to handle it differently
-            update_obj = utils.convertFlatObjectToNestedObject(update_obj);
-            exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
-            return true;
-        }
         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
@@ -376,14 +669,7 @@ exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
-            const props_to_update = Object.keys(update_obj);
-            for (let i = 0; i < props_to_update.length; i++) {
-                const prop_to_update = props_to_update[i];
-                const prop_value = update_obj[prop_to_update];
-                record[prop_to_update] = prop_value;
-            }
-        }).write();
         return true;
     }
 
@@ -391,19 +677,7 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
     return !!(output['result']['ok']);
 }
 
-exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
+exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
-    // local db override
-    if (using_local_db) {
-        const props_to_remove = Object.keys(remove_obj);
-        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
-        return true;
-    }
-
-    const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
-    return !!(output['result']['ok']);
-}
-
-exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
     // local db override
     if (using_local_db) {
         local_db.get(table).each((record) => {
@@ -817,14 +1091,6 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
                     } else if ('$ne' in filter_prop_value) {
                         filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
-                    } else if ('$lt' in filter_prop_value) {
-                        filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
-                    } else if ('$gt' in filter_prop_value) {
-                        filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
-                    } else if ('$lte' in filter_prop_value) {
-                        filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
-                    } else if ('$gte' in filter_prop_value) {
-                        filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
                     }
                 } else {
                     // handle case of nested property check
@@ -839,8 +1105,3 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     });
     return return_val;
 }
-
-// should only be used for tests
-exports.setLocalDBMode = (mode) => {
-    using_local_db = mode;
-}

backend/downloader.js

@@ -1,6 +1,7 @@
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const path = require('path');
+const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
@@ -13,9 +14,6 @@ const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
-const files_api = require('./files');
-const notifications_api = require('./notifications');
-const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
@@ -28,25 +26,6 @@ if (db_api.database_initialized) {
     });
 }
 
-/*
-
-This file handles all the downloading functionality.
-
-To download a file, we go through 4 steps. Here they are with their respective index & function:
-
-0: Create the download
- - createDownload()
-1: Get info for the download (we need this step for categories and archive functionality)
- - collectInfo()
-2: Download the file
- - downloadQueuedFile()
-3: Complete
- - N/A
-
-We use checkDownloads() to move downloads through the steps and call their respective functions.
-
-*/
-
 exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
@@ -105,10 +84,10 @@ exports.resumeDownload = async (download_uid) => {
 exports.restartDownload = async (download_uid) => {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     await exports.clearDownload(download_uid);
-    const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
+    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
     
     should_check_downloads = true;
-    return new_download;
+    return success;
 }
 
 exports.cancelDownload = async (download_uid) => {
@@ -127,10 +106,9 @@ exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download, error_message, error_type = null) {
+async function handleDownloadError(download_uid, error_message) {
-    if (!download || !download['uid']) return;
+    if (!download_uid) return;
-    notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
-    await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
 async function setupDownloads() {
@@ -176,13 +154,6 @@ async function checkDownloads() {
         if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
 
         if (waiting_download['finished_step'] && !waiting_download['finished']) {
-            if (waiting_download['sub_id']) {
-                const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
-                if (sub_missing) {
-                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
-                    continue;
-                }
-            }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
@@ -222,21 +193,6 @@ async function collectInfo(download_uid) {
         return;
     }
 
-    // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
-    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-    if (useYoutubeDLArchive && !options.ignoreArchive) {
-        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
-        if (exists_in_archive) {
-            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
-            logger.warn(error);
-            if (download_uid) {
-                const download = await db_api.getRecord('download_queue', {uid: download_uid});
-                await handleDownloadError(download, error, 'exists_in_archive');
-                return;
-            }
-        }
-    }
-
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -247,10 +203,11 @@ async function collectInfo(download_uid) {
         options.customOutput = category['custom_output'];
         options.noRelativePath = true;
         args = await exports.generateArgs(url, type, options, download['user_uid']);
+        args = utils.filterArgs(args, ['--no-simulate']);
         info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
-    const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
+    download['category'] = category;
 
     // setup info required to calculate download progress
 
@@ -273,7 +230,6 @@ async function collectInfo(download_uid) {
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
                                                                     title: playlist_title ? playlist_title : info['title'],
-                                                                    category: stripped_category,
                                                                     prefetched_info: null
                                                                 });
 }
@@ -316,14 +272,14 @@ async function downloadQueuedFile(download_uid) {
             clearInterval(download_checker);
             if (err) {
                 logger.error(err.stderr);
-                await handleDownloadError(download, err.stderr, 'unknown_error');
+                await handleDownloadError(download_uid, err.stderr);
                 resolve(false);
                 return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download, error_message, 'no_output');
+                    await handleDownloadError(download_uid, error_message);
                     logger.warn(error_message);
                     resolve(false);
                     return;
@@ -332,10 +288,7 @@ async function downloadQueuedFile(download_uid) {
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
-                        // we have to do this because sometimes there will be leading characters before the actual json
+                        output_json = JSON.parse(output[i]);
-                        const start_idx = output[i].indexOf('{"');
-                        const clean_output = output[i].slice(start_idx, output[i].length);
-                        output_json = JSON.parse(clean_output);
                     } catch(e) {
                         output_json = null;
                     }
@@ -352,7 +305,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -386,27 +339,31 @@ async function downloadQueuedFile(download_uid) {
                     }
 
                     // registers file in DB
-                    const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
-
-                    await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
-
-                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
 
                     file_objs.push(file_obj);
                 }
 
+                if (options.merged_string !== null && options.merged_string !== undefined) {
+                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
+                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
+                    const diff = current_merged_archive.replace(options.merged_string, '');
+                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+                    fs.appendFileSync(archive_path, diff);
+                }
+
                 let container = null;
 
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
                 } else {
                     const error_message = 'Downloaded file failed to result in metadata object.';
                     logger.error(error_message);
-                    await handleDownloadError(download, error_message, 'no_metadata');
+                    await handleDownloadError(download_uid, error_message);
                 }
 
                 const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -422,10 +379,6 @@ async function downloadQueuedFile(download_uid) {
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
 
-    if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
-        logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
-    }
-
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
     const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
@@ -473,8 +426,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
         } else if (heightParam && heightParam !== '' && !is_audio) {
-            const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
+            qualityPath = ['-f', `'(mp4)[height${maxHeight ? '<' : ''}=${heightParam}]`];
-            qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -511,6 +463,28 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
+            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+
+            await fs.ensureDir(archive_folder);
+            await fs.ensureFile(archive_path);
+
+            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
+            await fs.ensureFile(blacklist_path);
+
+            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
+            await fs.ensureFile(merged_path);
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            await mergeFiles(inputPathList, merged_path);
+
+            options.merged_string = await fs.readFile(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -553,8 +527,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
-        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...args];
-        const new_args = [...temp_args];
 
         const archiveArgIndex = new_args.indexOf('--download-archive');
         if (archiveArgIndex !== -1) {
@@ -586,8 +559,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                     const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
                     logger.error(error);
                     if (download_uid) {
-                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download_uid, error);
-                        await handleDownloadError(download, error, 'parse_failed');
                     }
                     resolve(null);
                 }
@@ -596,8 +568,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
                 if (err.stderr) error_message += `\n\n${err.stderr}`;
                 logger.error(error_message);
                 if (download_uid) {
-                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download_uid, error_message);
-                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
                 }
                 resolve(null);
             }
@@ -663,3 +634,13 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
+
+function getArchiveFolder(fileFolderPath, options, user_uid) {
+    if (options.customArchivePath) {
+        return path.join(options.customArchivePath);
+    } else if (user_uid) {
+        return path.join(fileFolderPath, 'archives');
+    } else {
+        return path.join('appdata', 'archives');
+    }
+}

backend/entrypoint.sh

@@ -1,7 +1,17 @@
-#!/bin/bash
+#!/bin/sh
 set -eu
 
+CMD="npm start"
+
+# if the first arg starts with "-" pass it to program
+if [ "${1#-}" != "$1" ]; then
+  set -- "$CMD" "$@"
+fi
+
 # chown current working directory to current user
-echo "[entrypoint] setup permission, this may take a while"
+if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
-find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
-exec gosu "$UID:$GID" "$@"
+  exec gosu "$UID:$GID" "$0" "$@"
+fi
+
+exec "$@"

backend/files.js

@@ -1,350 +0,0 @@
-const fs = require('fs-extra')
-const path = require('path')
-const { uuid } = require('uuidv4');
-
-const config_api = require('./config');
-const db_api = require('./db');
-const archive_api = require('./archive');
-const utils = require('./utils')
-const logger = require('./logger');
-
-exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
-    if (!file_object) file_object = generateFileObject(file_path, type);
-    if (!file_object) {
-        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
-        return false;
-    }
-
-    utils.fixVideoMetadataPerms(file_path, type);
-
-    // add thumbnail path
-    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
-
-    // if category exists, only include essential info
-    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
-
-    // modify duration
-    if (cropFileSettings) {
-        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
-    }
-
-    if (user_uid) file_object['user_uid'] = user_uid;
-    if (sub_id) file_object['sub_id'] = sub_id;
-
-    const file_obj = await registerFileDBManual(file_object);
-
-    // remove metadata JSON if needed
-    if (!config_api.getConfigItem('ytdl_include_metadata')) {
-        utils.deleteJSONFile(file_path, type)
-    }
-
-    return file_obj;
-}
-
-async function registerFileDBManual(file_object) {
-    // add additional info
-    file_object['uid'] = uuid();
-    file_object['registered'] = Date.now();
-    const path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
-
-    await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']})
-
-    return file_object;
-}
-
-function generateFileObject(file_path, type) {
-    const jsonobj = utils.getJSON(file_path, type);
-    if (!jsonobj) {
-        return null;
-    } else if (!jsonobj['_filename']) {
-        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
-        return null;
-    }
-    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
-    // console.
-    const stats = fs.statSync(true_file_path);
-
-    const file_id = utils.removeFileExtension(path.basename(file_path));
-    const title = jsonobj.title;
-    const url = jsonobj.webpage_url;
-    const uploader = jsonobj.uploader;
-    const upload_date = utils.formatDateString(jsonobj.upload_date);
-
-    const size = stats.size;
-
-    const thumbnail = jsonobj.thumbnail;
-    const duration = jsonobj.duration;
-    const isaudio = type === 'audio';
-    const description = jsonobj.description;
-    const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
-    return file_obj;
-}
-
-exports.importUnregisteredFiles = async () => {
-    const imported_files = [];
-    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
-
-    // run through check list and check each file to see if it's missing from the db
-    for (let i = 0; i < dirs_to_check.length; i++) {
-        const dir_to_check = dirs_to_check[i];
-        // recursively get all files in dir's path
-        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
-
-        for (let j = 0; j < files.length; j++) {
-            const file = files[j];
-
-            // check if file exists in db, if not add it
-            const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
-            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
-            if (!file_is_registered) {
-                // add additional info
-                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
-                if (file_obj) {
-                    imported_files.push(file_obj['uid']);
-                    logger.verbose(`Added discovered file to the database: ${file.id}`);
-                } else {
-                    logger.error(`Failed to import ${file['path']} automatically.`);
-                }
-            }
-        }
-    }
-    return imported_files;
-}
-
-exports.addMetadataPropertyToDB = async (property_key) => {
-    try {
-        const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
-        const update_obj = {};
-        for (let i = 0; i < dirs_to_check.length; i++) {
-            const dir_to_check = dirs_to_check[i];
-
-            // recursively get all files in dir's path
-            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
-            for (let j = 0; j < files.length; j++) {
-                const file = files[j];
-                if (file[property_key]) {
-                    update_obj[file.uid] = {[property_key]: file[property_key]};
-                }
-            }
-        }
-
-        return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj);
-    } catch(err) {
-        logger.error(err);
-        return false;
-    }
-}
-
-exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
-    const first_video = await exports.getVideo(uids[0]);
-    const thumbnailToUse = first_video['thumbnailURL'];
-    
-    let new_playlist = {
-        name: playlist_name,
-        uids: uids,
-        id: uuid(),
-        thumbnailURL: thumbnailToUse,
-        registered: Date.now(),
-        randomize_order: false
-    };
-
-    new_playlist.user_uid = user_uid ? user_uid : undefined;
-
-    await db_api.insertRecordIntoTable('playlists', new_playlist);
-    
-    const duration = await exports.calculatePlaylistDuration(new_playlist);
-    await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
-
-    return new_playlist;
-}
-
-exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
-    let playlist = await db_api.getRecord('playlists', {id: playlist_id});
-
-    if (!playlist) {
-        playlist = await db_api.getRecord('categories', {uid: playlist_id});
-        if (playlist) {
-            const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
-            playlist['uids'] = uids;
-            playlist['auto'] = true;
-        }
-    }
-
-    // converts playlists to new UID-based schema
-    if (playlist && playlist['fileNames'] && !playlist['uids']) {
-        playlist['uids'] = [];
-        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
-        for (let i = 0; i < playlist['fileNames'].length; i++) {
-            const fileName = playlist['fileNames'][i];
-            const uid = await exports.getVideoUIDByID(fileName, user_uid);
-            if (uid) playlist['uids'].push(uid);
-            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
-        }
-        exports.updatePlaylist(playlist, user_uid);
-    }
-
-    // prevent unauthorized users from accessing the file info
-    if (require_sharing && !playlist['sharingEnabled']) return null;
-
-    return playlist;
-}
-
-exports.updatePlaylist = async (playlist) => {
-    let playlistID = playlist.id;
-
-    const duration = await exports.calculatePlaylistDuration(playlist);
-    playlist.duration = duration;
-
-    return await db_api.updateRecord('playlists', {id: playlistID}, playlist);
-}
-
-exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
-    let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj);
-
-    if (!success) {
-        success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj);
-    }
-
-    if (!success) {
-        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
-    }
-
-    return success;
-}
-
-exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
-    if (!playlist_file_objs) {
-        playlist_file_objs = [];
-        for (let i = 0; i < playlist['uids'].length; i++) {
-            const uid = playlist['uids'][i];
-            const file_obj = await exports.getVideo(uid);
-            if (file_obj) playlist_file_objs.push(file_obj);
-        }
-    }
-
-    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
-}
-
-exports.deleteFile = async (uid, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid);
-    const type = file_obj.isAudio ? 'audio' : 'video';
-    const folderPath = path.dirname(file_obj.path);
-    const name = file_obj.id;
-    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
-
-    var jsonPath = `${file_obj.path}.info.json`;
-    var altJSONPath = `${filePathNoExtension}.info.json`;
-    var thumbnailPath = `${filePathNoExtension}.webp`;
-    var altThumbnailPath = `${filePathNoExtension}.jpg`;
-
-    jsonPath = path.join(__dirname, jsonPath);
-    altJSONPath = path.join(__dirname, altJSONPath);
-
-    let jsonExists = await fs.pathExists(jsonPath);
-    let thumbnailExists = await fs.pathExists(thumbnailPath);
-
-    if (!jsonExists) {
-        if (await fs.pathExists(altJSONPath)) {
-            jsonExists = true;
-            jsonPath = altJSONPath;
-        }
-    }
-
-    if (!thumbnailExists) {
-        if (await fs.pathExists(altThumbnailPath)) {
-            thumbnailExists = true;
-            thumbnailPath = altThumbnailPath;
-        }
-    }
-
-    let fileExists = await fs.pathExists(file_obj.path);
-
-    if (config_api.descriptors[uid]) {
-        try {
-            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
-                config_api.descriptors[uid][i].destroy();
-            }
-        } catch(e) {
-
-        }
-    }
-
-    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-    if (useYoutubeDLArchive || file_obj.sub_id) {
-        // get id/extractor from JSON
-
-        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let retrievedID = null;
-        let retrievedExtractor = null;
-        if (info_json) {
-            retrievedID = info_json['id'];
-            retrievedExtractor = info_json['extractor'];
-        }
-
-        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-        if (!blacklistMode) {
-            await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id)
-        } else {
-            const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
-            if (!exists_in_archive) {
-                await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id);
-            }
-        }
-    }
-
-    if (jsonExists) await fs.unlink(jsonPath);
-    if (thumbnailExists) await fs.unlink(thumbnailPath);
-
-    await db_api.removeRecord('files', {uid: uid});
-
-    if (fileExists) {
-        await fs.unlink(file_obj.path);
-        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
-            return false;
-        } else {
-            return true;
-        }
-    } else {
-        // TODO: tell user that the file didn't exist
-        return true;
-    }
-}
-
-// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
-exports.getVideoUIDByID = async (file_id, uuid = null) => {
-    const file_obj = await db_api.getRecord('files', {id: file_id});
-    return file_obj ? file_obj['uid'] : null;
-}
-
-exports.getVideo = async (file_uid) => {
-    return await db_api.getRecord('files', {uid: file_uid});
-}
-
-exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
-    const filter_obj = {user_uid: uuid};
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (favorite_filter) {
-        filter_obj['favorite'] = true;
-    }
-
-    if (sub_id) {
-        filter_obj['sub_id'] = sub_id;
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search)));
-    const file_count = await db_api.getRecords('files', filter_obj, true);
-
-    return {files, file_count};
-}

backend/notifications.js

@@ -1,249 +0,0 @@
-const db_api = require('./db');
-const config_api = require('./config');
-const logger = require('./logger');
-const utils = require('./utils');
-const consts = require('./consts');
-
-const { uuid } = require('uuidv4');
-
-const fetch = require('node-fetch');
-const { gotify } = require("gotify");
-const TelegramBot = require('node-telegram-bot-api');
-const REST = require('@discordjs/rest').REST;
-const API = require('@discordjs/core').API;
-const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
-
-const NOTIFICATION_TYPE_TO_TITLE = {
-    task_finished: 'Task finished',
-    download_complete: 'Download complete',
-    download_error: 'Download error'
-}
-
-const NOTIFICATION_TYPE_TO_BODY = {
-    task_finished: (notification) => notification['data']['task_title'],
-    download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
-    download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
-}
-
-const NOTIFICATION_TYPE_TO_URL = {
-    task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
-    download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
-    download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
-}
-
-const NOTIFICATION_TYPE_TO_THUMBNAIL = {
-    task_finished: () => null,
-    download_complete: (notification) => notification['data']['file_thumbnail'],
-    download_error: () => null
-}
-
-exports.sendNotification = async (notification) => {
-    // info necessary if we are using 3rd party APIs
-    const type = notification['type'];
-
-    const data = {
-        title: NOTIFICATION_TYPE_TO_TITLE[type],
-        body: NOTIFICATION_TYPE_TO_BODY[type](notification),
-        type: type,
-        url: NOTIFICATION_TYPE_TO_URL[type](notification),
-        thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
-    }
-
-    if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
-        sendNtfyNotification(data);
-    }
-    if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
-        sendGotifyNotification(data);
-    }
-    if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
-        sendTelegramNotification(data);
-    }
-    if (config_api.getConfigItem('ytdl_webhook_url')) {
-        sendGenericNotification(data);
-    }
-    if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
-        sendDiscordNotification(data);
-    }
-    if (config_api.getConfigItem('ytdl_slack_webhook_url')) {
-        sendSlackNotification(data);
-    }
-
-    await db_api.insertRecordIntoTable('notifications', notification);
-    return notification;
-}
-
-exports.sendTaskNotification = async (task_obj, confirmed) => {
-    if (!notificationEnabled('task_finished')) return;
-    // workaround for tasks which are user_uid agnostic
-    const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
-    await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
-    const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
-    const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
-    return await exports.sendNotification(notification);
-}
-
-exports.sendDownloadNotification = async (file, user_uid) => {
-    if (!notificationEnabled('download_complete')) return;
-    const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
-    const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
-    return await exports.sendNotification(notification);
-}
-
-exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
-    if (!notificationEnabled('download_error')) return;
-    const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
-    const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
-    return await exports.sendNotification(notification);
-}
-
-exports.createNotification = (type, actions, data, user_uid) => {
-    const notification = {
-        type: type,
-        actions: actions,
-        data: data,
-        user_uid: user_uid,
-        uid: uuid(),
-        read: false,
-        timestamp: Date.now()/1000
-    }
-    return notification;
-}
-
-function notificationEnabled(type) {
-    return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
-}
-
-function sendNtfyNotification({body, title, type, url, thumbnail}) {
-    logger.verbose('Sending notification to ntfy');
-    fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
-        method: 'POST',
-        body: body,
-        headers: {
-            'Title': title,
-            'Tags': type,
-            'Click': url,
-            'Attach': thumbnail
-        }
-    });
-}
-
-async function sendGotifyNotification({body, title, type, url, thumbnail}) {
-    logger.verbose('Sending notification to gotify');
-    await gotify({
-        server: config_api.getConfigItem('ytdl_gotify_server_url'),
-        app: config_api.getConfigItem('ytdl_gotify_app_token'),
-        title: title,
-        message: body,
-        tag: type,
-        priority: 5, // Keeping default from docs, may want to change this,
-        extras: {
-            "client::notification": {
-                click: { url: url },
-                bigImageUrl: thumbnail
-            }
-        }
-      });
-}
-
-async function sendTelegramNotification({body, title, type, url, thumbnail}) {
-    logger.verbose('Sending notification to Telegram');
-    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
-    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
-    const bot = new TelegramBot(bot_token);
-    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
-    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
-}
-
-async function sendDiscordNotification({body, title, type, url, thumbnail}) {
-    const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
-    const url_split = discord_webhook_url.split('webhooks/');
-    const [webhook_id, webhook_token] = url_split[1].split('/');
-    const rest = new REST({ version: '10' });
-    const api = new API(rest);
-    const embed = new EmbedBuilder()
-        .setTitle(title)
-        .setColor(0x00FFFF)
-        .setURL(url)
-        .setDescription(`ID: ${type}`);
-    if (thumbnail) embed.setThumbnail(thumbnail);
-    if (type === 'download_error') embed.setColor(0xFC2003);
-
-    const result = await api.webhooks.execute(webhook_id, webhook_token, {
-        content: body,
-        username: 'YoutubeDL-Material',
-        avatar_url: consts.ICON_URL,
-        embeds: [embed],
-    });
-    return result;
-}
-
-function sendSlackNotification({body, title, type, url, thumbnail}) {
-    const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
-    logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
-    const data = {
-        blocks: [
-            {
-                type: "section",
-                text: {
-                    type: "mrkdwn",
-                    text: `*${title}*`
-                }
-            },
-            {
-                type: "section",
-                text: {
-                    type: "plain_text",
-                    text: body
-                }
-            }
-        ]
-    }
-
-    // add thumbnail if exists
-    if (thumbnail) {
-        data['blocks'].push({
-            type: "image",
-            image_url: thumbnail,
-            alt_text: "notification_thumbnail"
-        });
-    }
-
-    data['blocks'].push(
-        {
-            type: "section",
-            text: {
-                type: "mrkdwn",
-                text: `<${url}|${url}>`
-            }
-        },
-        {
-            type: "context",
-            elements: [
-                {
-                    type: "mrkdwn",
-                    text: `*ID:* ${type}`
-                }
-            ]
-        }
-    );
-
-    fetch(slack_webhook_url, {
-        method: 'POST',
-        headers: {
-            "Content-Type": "application/json"
-        },
-        body: JSON.stringify(data),
-    });
-}
-
-function sendGenericNotification(data) {
-    const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
-    logger.verbose(`Sending generic notification to ${webhook_url}`);
-    fetch(webhook_url, {
-        method: 'POST',
-        headers: {
-            "Content-Type": "application/json"
-        },
-        body: JSON.stringify(data),
-    });
-}

backend/package.json

@@ -17,43 +17,32 @@
   "bugs": {
     "url": ""
   },
-  "engines": {
-    "node": "^16",
-    "npm": "6.14.4"
-  },
   "homepage": "",
   "dependencies": {
-    "@discordjs/builders": "^1.6.1",
-    "@discordjs/core": "^0.5.2",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
-    "async-mutex": "^0.4.0",
+    "async-mutex": "^0.3.1",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
-    "command-exists": "^1.2.9",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.18.2",
+    "express": "^4.17.3",
-    "express-session": "^1.17.3",
-    "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
-    "gotify": "^1.1.0",
+    "jsonwebtoken": "^8.5.1",
-    "jsonwebtoken": "^9.0.0",
-    "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
+    "merge-files": "^0.1.2",
     "mocha": "^9.2.2",
-    "moment": "^2.29.4",
+    "moment": "^2.29.2",
     "mongodb": "^3.6.9",
     "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
-    "node-id3": "^0.2.6",
+    "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
-    "node-telegram-bot-api": "^0.61.0",
+    "passport": "^0.4.1",
-    "passport": "^0.6.0",
     "passport-http": "^0.3.0",
-    "passport-jwt": "^4.0.1",
+    "passport-jwt": "^4.0.0",
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
@@ -62,7 +51,7 @@
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.2.13",
+    "uuidv4": "^6.0.6",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"

backend/subscriptions.js

@@ -3,24 +3,22 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
-const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
-const CONSTS = require('./consts');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 const db_api = require('./db');
 const downloader_api = require('./downloader');
 
-exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
+async function subscribe(sub, user_uid = null) {
     const result_obj = {
         success: false,
         error: ''
     };
     return new Promise(async resolve => {
         // sub should just have url and name. here we will get isPlaylist and path
-        sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
+        sub.isPlaylist = sub.url.includes('playlist');
         sub.videos = [];
 
         let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -35,11 +33,10 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
         sub['user_uid'] = user_uid ? user_uid : undefined;
         await db_api.insertRecordIntoTable('subscriptions', sub);
 
-        let success = skip_get_info ? true : await getSubscriptionInfo(sub);
+        let success = await getSubscriptionInfo(sub);
-        exports.writeSubscriptionMetadata(sub);
 
         if (success) {
-            if (!sub.paused) exports.getVideosForSub(sub, user_uid);
+            getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
         }
@@ -94,10 +91,7 @@ async function getSubscriptionInfo(sub) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            let sub_name = sub.name;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
-                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
-                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
-                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
                         }
                     }
 
@@ -111,7 +105,7 @@ async function getSubscriptionInfo(sub) {
     });
 }
 
-exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
+async function unsubscribe(sub, deleteMode, user_uid = null) {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -144,25 +138,28 @@ exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
 
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     if (deleteMode && (await fs.pathExists(appendedBasePath))) {
+        if (sub.archive && (await fs.pathExists(sub.archive))) {
+            const archive_file_path = path.join(sub.archive, 'archive.txt');
+            // deletes archive if it exists
+            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
+            if (await fs.pathExists(archive_file_path)) {
+                await fs.unlink(archive_file_path);
+            }
+            await fs.rmdir(sub.archive);
+        }
         await fs.remove(appendedBasePath);
     }
-
-    await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
-exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
+async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
-    if (typeof sub === 'string') {
-        // TODO: fix bad workaround where sub is a sub_id
-        sub = await db_api.getRecord('subscriptions', {sub_id: sub});
-    }
     // TODO: combine this with deletefile
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
-    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -181,9 +178,7 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
     ]);
 
     if (jsonExists) {
-        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = fs.readJSONSync(jsonPath)['id'];
-        retrievedID = info_json['id'];
-        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -201,14 +196,11 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (deleteForever) {
+            if (useArchive && retrievedID) {
-                // ensure video is in the archives
+                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
-                const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
+
-                if (!exists_in_archive) {
+                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-                    await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
+                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
-                }
-            } else {
-                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
         }
@@ -218,8 +210,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
     }
 }
 
-exports.getVideosForSub = async (sub, user_uid = null) => {
+async function getVideosForSub(sub, user_uid = null) {
-    const latest_sub_obj = await exports.getSubscription(sub.id);
+    const latest_sub_obj = await getSubscription(sub.id);
     if (!latest_sub_obj || latest_sub_obj['downloading']) {
         return false;
     }
@@ -239,20 +231,13 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
-            // remove temporary archive file if it exists
-            const archive_path = path.join(appendedBasePath, 'archive.txt');
-            const archive_exists = await fs.pathExists(archive_path);
-            if (archive_exists) {
-                await fs.unlink(archive_path);
-            }
-
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
@@ -307,7 +292,7 @@ async function handleOutputJSON(output, sub, user_uid) {
     }
 
     const files_to_download = await getFilesToDownload(sub, output_jsons);
-    const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
 
     for (let j = 0; j < files_to_download.length; j++) {
         const file_to_download = files_to_download[j];
@@ -318,7 +303,7 @@ async function handleOutputJSON(output, sub, user_uid) {
     return files_to_download;
 }
 
-exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
+function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -346,6 +331,8 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
+    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -371,16 +358,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
-    // skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
-    const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
-    const archive_count = archive_text.split('\n').length - 1;
-    if (archive_count > 0) {
-        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
-        const archive_path = path.join(appendedBasePath, 'archive.txt');
-        await fs.writeFile(archive_path, archive_text);
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
@@ -391,6 +368,21 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
+    let archive_dir = null;
+    let archive_path = null;
+
+    if (useArchive && !redownload) {
+        if (sub.archive) {
+            archive_dir = sub.archive;
+            if (sub.type && sub.type === 'audio') {
+                archive_path = path.join(archive_dir, 'merged_audio.txt');
+            } else {
+                archive_path = path.join(archive_dir, 'merged_video.txt');
+            }            
+        }
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -433,11 +425,7 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
-                continue;
             }
-            const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
-            if (exists_in_archive) continue;
-
             files_to_download.push(output_json);
         }
     }
@@ -445,36 +433,30 @@ async function getFilesToDownload(sub, output_jsons) {
 }
 
 
-exports.getSubscriptions = async (user_uid = null) => {
+async function getSubscriptions(user_uid = null) {
     return await db_api.getRecords('subscriptions', {user_uid: user_uid});
 }
 
-exports.getAllSubscriptions = async () => {
+async function getAllSubscriptions() {
     const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
 }
 
-exports.getSubscription = async (subID) => {
+async function getSubscription(subID) {
-    // stringify and parse because we may override the 'downloading' property
+    return await db_api.getRecord('subscriptions', {id: subID});
-    const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
-    // now with the download_queue, we may need to override 'downloading'
-    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
-    if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
-    return sub;
 }
 
-exports.getSubscriptionByName = async (subName, user_uid = null) => {
+async function getSubscriptionByName(subName, user_uid = null) {
     return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
 }
 
-exports.updateSubscription = async (sub) => {
+async function updateSubscription(sub) {
     await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
-    exports.writeSubscriptionMetadata(sub);
     return true;
 }
 
-exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
+async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
         await updateSubscriptionProperty(sub, assignment_obj);
     });
@@ -486,14 +468,6 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-exports.writeSubscriptionMetadata = (sub) => {
-    let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
-                                : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const appendedBasePath = getAppendedBasePath(sub, basePath);
-    const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
-    fs.writeJSONSync(metadata_path, sub);
-}
-
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     if (!sub_files) return;
@@ -548,3 +522,17 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
 function getAppendedBasePath(sub, base_path) {
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
+
+module.exports = {
+    getSubscription        : getSubscription,
+    getSubscriptionByName  : getSubscriptionByName,
+    getSubscriptions       : getSubscriptions,
+    getAllSubscriptions    : getAllSubscriptions,
+    updateSubscription     : updateSubscription,
+    subscribe              : subscribe,
+    unsubscribe            : unsubscribe,
+    deleteSubscriptionFile : deleteSubscriptionFile,
+    getVideosForSub        : getVideosForSub,
+    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
+    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
+}

backend/tasks.js

@@ -1,19 +1,9 @@
 const db_api = require('./db');
-const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
-const archive_api = require('./archive');
-const files_api = require('./files');
-const subscriptions_api = require('./subscriptions');
-const config_api = require('./config');
-const auth_api = require('./authentication/auth');
-const utils = require('./utils');
-const logger = require('./logger');
-const CONSTS = require('./consts');
 
 const fs = require('fs-extra');
-const path = require('path');
+const logger = require('./logger');
 const scheduler = require('node-schedule');
-const { uuid } = require('uuidv4');
 
 const TASKS = {
     backup_local_db: {
@@ -28,7 +18,7 @@ const TASKS = {
         job: null
     },
     missing_db_records: {
-        run: files_api.importUnregisteredFiles,
+        run: db_api.importUnregisteredFiles,
         title: 'Import missing DB records',
         job: null
     },
@@ -43,33 +33,6 @@ const TASKS = {
         confirm: youtubedl_api.updateYoutubeDL,
         title: 'Update youtube-dl',
         job: null
-    },
-    delete_old_files: {
-        run: checkForAutoDeleteFiles,
-        confirm: autoDeleteFiles,
-        title: 'Delete old files',
-        job: null
-    },
-    import_legacy_archives: {
-        run: archive_api.importArchives,
-        title: 'Import legacy archives',
-        job: null
-    },
-    rebuild_database: {
-        run: rebuildDB,
-        title: 'Rebuild database',
-        job: null
-    }
-}
-
-const defaultOptions = {
-    all: {
-        auto_confirm: false
-    },
-    delete_old_files: {
-        blacklist_files: false,
-        blacklist_subscription_files: false,
-        threshold_days: ''
     }
 }
 
@@ -82,7 +45,7 @@ function scheduleJob(task_key, schedule) {
         const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
         const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
         const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
         return null;
@@ -94,7 +57,7 @@ function scheduleJob(task_key, schedule) {
             logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
             return;
         }
-
+        
         // remove schedule if it's a one-time task
         if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
         // we're just "running" the task, any confirmation should be user-initiated
@@ -114,10 +77,9 @@ exports.setupTasks = async () => {
     const tasks_keys = Object.keys(TASKS);
     for (let i = 0; i < tasks_keys.length; i++) {
         const task_key = tasks_keys[i];
-        const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
         const task_in_db = await db_api.getRecord('tasks', {key: task_key});
         if (!task_in_db) {
-            // insert task metadata into table if missing, eventually move title to UI
+            // insert task metadata into table if missing
             await db_api.insertRecordIntoTable('tasks', {
                 key: task_key,
                 title: TASKS[task_key]['title'],
@@ -128,19 +90,9 @@ exports.setupTasks = async () => {
                 data: null,
                 error: null,
                 schedule: null,
-                options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
+                options: {}
             });
         } else {
-            // verify all options exist in task
-            for (const key of Object.keys(mergedDefaultOptions)) {
-                const option_key = `options.${key}`;
-                // Remove any potential mangled option keys (#861)
-                await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
-                if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
-                    await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
-                }
-            }
-
             // reset task if necessary
             await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
 
@@ -171,23 +123,15 @@ exports.executeTask = async (task_key) => {
 
 exports.executeRun = async (task_key) => {
     logger.verbose(`Running task ${task_key}`);
-    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     // don't set running to true when backup up DB as it will be stick "running" if restored
     if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
     const data = await TASKS[task_key].run();
     await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
     logger.verbose(`Finished running task ${task_key}`);
-    const task_obj = await db_api.getRecord('tasks', {key: task_key});
-    await notifications_api.sendTaskNotification(task_obj, false);
-
-    if (task_obj['options'] && task_obj['options']['auto_confirm']) {
-        exports.executeConfirm(task_key);
-    }
 }
 
 exports.executeConfirm = async (task_key) => {
     logger.verbose(`Confirming task ${task_key}`);
-    await db_api.updateRecord('tasks', {key: task_key}, {error: null})
     if (!TASKS[task_key]['confirm']) {
         return null;
     }
@@ -197,7 +141,6 @@ exports.executeConfirm = async (task_key) => {
     await TASKS[task_key].confirm(data);
     await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
     logger.verbose(`Finished confirming task ${task_key}`);
-    await notifications_api.sendTaskNotification(task_obj, false);
 }
 
 exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -250,95 +193,4 @@ async function removeDuplicates(data) {
     }
 }
 
-// auto delete files
-
-async function checkForAutoDeleteFiles() {
-    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
-    if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
-        const error_message = 'Failed to do delete check because no limit was set!';
-        logger.error(error_message);
-        await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
-        return null;
-    }
-    const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
-    const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
-    const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
-    return {files_to_remove: files_to_remove};
-}
-
-async function autoDeleteFiles(data) {
-    const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
-    if (data['files_to_remove']) {
-        logger.info(`Removing ${data['files_to_remove'].length} old files!`);
-        for (let i = 0; i < data['files_to_remove'].length; i++) {
-            const file_to_remove = data['files_to_remove'][i];
-            await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
-        }
-    }
-}
-
-async function rebuildDB() {
-    await db_api.backupDB();
-    let subs_to_add = await guessSubscriptions(false);
-    subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
-    const users_to_add = await guessUsers();
-    for (const user_to_add of users_to_add) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        
-        const user_exists = await db_api.getRecord('users', {uid: user_to_add});
-        if (!user_exists) {
-            await auth_api.registerUser(user_to_add, user_to_add, 'password');
-            logger.info(`Regenerated user ${user_to_add}`);
-        }
-        
-        const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
-        const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
-        subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
-    }
-
-    for (const sub_to_add of subs_to_add) {
-        const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
-        // TODO: we shouldn't be creating this here
-        const new_sub = Object.assign({}, sub_to_add, {paused: true});
-        if (!sub_exists) {
-            await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
-            logger.info(`Regenerated subscription ${sub_to_add['name']}`);
-        }
-    }
-
-    logger.info(`Importing unregistered files`);
-    await files_api.importUnregisteredFiles();
-}
-
-const guessUsers = async () => {
-    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-    const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
-    return userPaths.map(userPath => path.basename(userPath));
-}
-
-const guessSubscriptions = async (isPlaylist, basePath = null) => {
-    const guessed_subs = [];
-    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
-
-    const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
-    const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
-
-    const subs = await utils.getDirectoriesInDirectory(subsPath);
-    for (const subPath of subs) {
-        const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
-        if (!fs.existsSync(sub_backup_path)) continue;
-
-        try {
-            const sub_backup = fs.readJSONSync(sub_backup_path)
-            delete sub_backup['_id'];
-            guessed_subs.push(sub_backup);
-        } catch(err) {
-            logger.warn(`Failed to reimport subscription in path ${subPath}`)
-            logger.warn(err);
-        }
-    }
-
-    return guessed_subs;
-}
-
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -1,9 +1,9 @@
-/* eslint-disable no-undef */
 const assert = require('assert');
 const low = require('lowdb')
 const winston = require('winston');
 const path = require('path');
 
+process.chdir('./backend')
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -38,9 +38,6 @@ var auth_api = require('../authentication/auth');
 var db_api = require('../db');
 const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
-const archive_api = require('../archive');
-const categories_api = require('../categories');
-const files_api = require('../files');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 const NodeID3 = require('node-id3');
@@ -69,12 +66,12 @@ const sample_video_json = {
 
 describe('Database', async function() {
     describe('Import', async function() {
-        // it('Migrate', async function() {
+        it('Migrate', async function() {
-        //     await db_api.connectToDB();
+            await db_api.connectToDB();
-        //     await db_api.removeAllRecords();
+            await db_api.removeAllRecords();
-        //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
+            const success = await db_api.importJSONToDB(db.value(), users_db.value());
-        //     assert(success);
+            assert(success);
-        // });
+        });
 
         it('Transfer to remote', async function() {
             await db_api.removeAllRecords('test');
@@ -107,208 +104,157 @@ describe('Database', async function() {
         });
     });
 
+    describe('Export', function() {
+
+    });
+
+
     describe('Basic functions', async function() {
-        
+        beforeEach(async function() {
-        // test both local_db and remote_db
+            await db_api.connectToDB();
-        const local_db_modes = [false, true];
+            await db_api.removeAllRecords('test');
+        });
+        it('Add and read record', async function() {
+            this.timeout(120000);
+            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+            assert(added_record['test_add'] === 'test');
+            await db_api.removeRecord('test', {test_add: 'test'});
+        });
 
-        for (const local_db_mode of local_db_modes) {
+        it('Find duplicates by key', async function() {
-            let use_local_db = local_db_mode;
+            const test_duplicates = [
-            describe(`Use local DB - ${use_local_db}`, async function() {
+                {
-                beforeEach(async function() {
+                    test: 'testing',
-                    if (!use_local_db) {
+                    key: '1'
-                        this.timeout(120000);
+                },
-                        await db_api.connectToDB(0);
+                {
-                    }
+                    test: 'testing',
-                    await db_api.removeAllRecords('test');
+                    key: '2'
+                },
+                {
+                    test: 'testing_missing',
+                    key: '3'
+                },
+                {
+                    test: 'testing',
+                    key: '4'
+                }
+            ];
+            await db_api.insertRecordsIntoTable('test', test_duplicates);
+            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+            console.log(duplicates);
+        });
+
+        it('Update record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+            assert(updated_record['added_field']);
+            await db_api.removeRecord('test', {test_update: 'test'});
+        });
+
+        it('Remove record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+            assert(delete_succeeded);
+            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+            assert(!deleted_record);
+        });
+
+        it('Push to record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 1);
+        });
+
+        it('Pull from record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 0);
+        });
+
+        it('Bulk add', async function() {
+            this.timeout(120000);
+            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+            const test_records = [];
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                test_records.push({
+                    uid: uuid()
                 });
-                it('Add and read record', async function() {
+            }
-                    this.timeout(120000);
+            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+
-                    const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+            const received_records = await db_api.getRecords('test');
-                    assert(added_record['test_add'] === 'test');
+            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
-                    await db_api.removeRecord('test', {test_add: 'test'});
+        });
-                });
+
-                it('Add and read record - Nested property', async function() {
+        it('Bulk update', async function() {
-                    this.timeout(120000);
+            // bulk add records
-                    await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
+            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
-                    const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
+            const test_records = [];
-                    const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
+            const update_obj = {};
-                    assert(added_record['test_add'] === 'test');
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                    assert(!not_added_record);
+                const test_uid =  uuid();
-                    await db_api.removeRecord('test', {test_add: 'test'});
+                test_records.push({
-                });
+                    uid: test_uid
-                it('Replace filter', async function() {
-                    this.timeout(120000);
-                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
-                    await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
-                    const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
-                    assert(count === 1);
-                    await db_api.removeRecord('test', {test_replace_filter: 'test'});
-                });
-                it('Find duplicates by key', async function() {
-                    const test_duplicates = [
-                        {
-                            test: 'testing',
-                            key: '1'
-                        },
-                        {
-                            test: 'testing',
-                            key: '2'
-                        },
-                        {
-                            test: 'testing_missing',
-                            key: '3'
-                        },
-                        {
-                            test: 'testing',
-                            key: '4'
-                        }
-                    ];
-                    await db_api.insertRecordsIntoTable('test', test_duplicates);
-                    const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-                    console.log(duplicates);
                 });
+                update_obj[test_uid] = {added_field: true};
+            }
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            assert(success);
 
-                it('Update record', async function() {
+            // makes sure they are added
-                    await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+            const received_records = await db_api.getRecords('test');
-                    await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
-                    const updated_record = await db_api.getRecord('test', {test_update: 'test'});
-                    assert(updated_record['added_field']);
-                    await db_api.removeRecord('test', {test_update: 'test'});
-                });
 
-                it('Update records', async function() {
+            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
-                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
+            assert(success);
-                    await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
-                    await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
-                    const updated_records = await db_api.getRecords('test', {added_field: true});
-                    assert(updated_records.length === 2);
-                    await db_api.removeRecord('test', {test_update: 'test'});
-                });
 
-                it('Remove property from record', async function() {
+            const received_updated_records = await db_api.getRecords('test');
-                    await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
+            for (let i = 0; i < received_updated_records.length; i++) {
-                    await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
+                success &= received_updated_records[i]['added_field'];
-                    const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
+            }
-                    assert(updated_record['test_keep']);
+            assert(success);
-                    assert(!updated_record['test_remove']);
+        });
-                    await db_api.removeRecord('test', {test_keep: 'test'});
-                });
 
-                it('Remove record', async function() {
+        it('Stats', async function() {
-                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+            const stats = await db_api.getDBStats();
-                    const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+            assert(stats);
-                    assert(delete_succeeded);
+        });
-                    const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
-                    assert(!deleted_record);
-                });
 
-                it('Remove records', async function() {
+        it('Query speed', async function() {
-                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
+            this.timeout(120000); 
-                    await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
+            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
-                    await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+            const test_records = [];
-                    const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
+            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
-                    assert(delete_succeeded);
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                    const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
+                const uid = uuid();
-                    assert(count === 0);
+                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                });
+                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+            }
+            const insert_start = Date.now();
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            const insert_end = Date.now();
 
-                it('Push to record array', async function() {
+            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
-                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
-                    await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
-                    const record = await db_api.getRecord('test', {test: 'test'});
-                    assert(record);
-                    assert(record['test_array'].length === 1);
-                });
 
-                it('Pull from record array', async function() {
+            const query_start = Date.now();
-                    await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+            const random_record = await db_api.getRecord('test', {uid: random_uid});
-                    await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const query_end = Date.now();
-                    const record = await db_api.getRecord('test', {test: 'test'});
-                    assert(record);
-                    assert(record['test_array'].length === 0);
-                });
 
-                it('Bulk add', async function() {
+            console.log(random_record)
-                    this.timeout(120000);
-                    const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
-                    const test_records = [];
-                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                        test_records.push({
-                            uid: uuid()
-                        });
-                    }
-                    const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
 
-                    const received_records = await db_api.getRecords('test');
+            console.log(`Query time: ${(query_end - query_start)/1000}s`);
-                    assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
-                });
 
-                it('Bulk update', async function() {
+            success = !!random_record;
-                    // bulk add records
-                    const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
-                    const test_records = [];
-                    const update_obj = {};
-                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                        const test_uid =  uuid();
-                        test_records.push({
-                            uid: test_uid
-                        });
-                        update_obj[test_uid] = {added_field: true};
-                    }
-                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-                    assert(success);
 
-                    // makes sure they are added
+            assert(success);
-                    const received_records = await db_api.getRecords('test');
+        });
-                    assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
-
-                    success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
-                    assert(success);
-
-                    const received_updated_records = await db_api.getRecords('test');
-                    for (let i = 0; i < received_updated_records.length; i++) {
-                        success &= received_updated_records[i]['added_field'];
-                    }
-                    assert(success);
-                });
-
-                it('Stats', async function() {
-                    const stats = await db_api.getDBStats();
-                    assert(stats);
-                });
-
-                it('Query speed', async function() {
-                    this.timeout(120000); 
-                    const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
-                    const test_records = [];
-                    let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
-                    for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
-                        const uid = uuid();
-                        if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                        test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
-                    }
-                    const insert_start = Date.now();
-                    let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
-                    const insert_end = Date.now();
-
-                    console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
-
-                    const query_start = Date.now();
-                    const random_record = await db_api.getRecord('test', {uid: random_uid});
-                    const query_end = Date.now();
-
-                    console.log(random_record)
-
-                    console.log(`Query time: ${(query_end - query_start)/1000}s`);
-
-                    success = !!random_record;
-
-                    assert(success);
-                });
-            });
-        }
     });
 
     describe('Local DB Filters', async function() {
@@ -337,56 +283,50 @@ describe('Database', async function() {
 });
 
 describe('Multi User', async function() {
-    const user_to_test = 'test_user';
+    let user = null;
-    const user_password = 'test_pass';
+    const user_to_test = 'admin';
-    const sub_to_test = '';
+    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
-    const playlist_to_test = '';
+    const playlist_to_test = 'ysabVZz4x';
     beforeEach(async function() {
         await db_api.connectToDB();
-        await auth_api.deleteUser(user_to_test);
+        auth_api.initialize(db_api, logger);
+        subscriptions_api.initialize(db_api, logger);
+        user = await auth_api.login('admin', 'pass');
     });
-    describe('Basic', function() {
+    describe('Authentication', function() {
-        it('Register', async function() {
+        it('login', async function() {
-            const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
-            assert(user);
-        });
-        it('Login', async function() {
-            await auth_api.registerUser(user_to_test, user_to_test, user_password);
-            const user = await auth_api.login(user_to_test, user_password);
             assert(user);
         });
     });
     describe('Video player - normal', async function() {
-        beforeEach(async function() {
+        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
-            await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+        await db_api.insertRecordIntoTable('files', sample_video_json);
-            await db_api.insertRecordIntoTable('files', sample_video_json);
-        });
         const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
-            const video_obj = await files_api.getVideo(video_to_test);
+            const video_obj = await db_api.getVideo(video_to_test);
             assert(video_obj);
         });
 
         it('Video access - disallowed', async function() {
-            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
-            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
             assert(!video_obj);
         });
 
         it('Video access - allowed', async function() {
             await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
-            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
             assert(video_obj);
         });
     });
     describe('Zip generators', function() {
         it('Playlist zip generator', async function() {
-            const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
+            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
             assert(playlist);
             const playlist_files_to_download = [];
             for (let i = 0; i < playlist['uids'].length; i++) {
                 const uid = playlist['uids'][i];
-                const playlist_file = await files_api.getVideo(uid, user_to_test);
+                const playlist_file = await db_api.getVideo(uid, user_to_test);
                 playlist_files_to_download.push(playlist_file);
             }
             const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
@@ -414,7 +354,7 @@ describe('Multi User', async function() {
     //     const sub_to_test = '';
     //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
     //     it('Get video', async function() {
-    //         const video_obj = files_api.getVideo(video_to_test, 'admin', );
+    //         const video_obj = db_api.getVideo(video_to_test, 'admin', );
     //         assert(video_obj);
     //     });
 
@@ -517,23 +457,18 @@ describe('Downloader', function() {
         const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
         const updated_args1 = utils.injectArgs(original_args1, new_args1);
         const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
-        assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
+        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
 
         const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
         const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
         const updated_args2 = utils.injectArgs(original_args2, new_args2);
-        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
-        assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
+        console.log(updated_args2);
-
+        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
-        const original_args3 = ['-o', '%(title)s.%(ext)s'];
-        const new_args3 =  ['--min-filesize','1'];
-        const updated_args3 = utils.injectArgs(original_args3, new_args3);
-        const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
-        assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
     });
     describe('Twitch', async function () {
         const twitch_api = require('../twitch');
-        const example_vod = '1710641401';
+        const example_vod = '1493770675';
         it('Download VOD', async function() {
             const sample_path = path.join('test', 'sample.twitch_chat.json');
             if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
@@ -615,7 +550,7 @@ describe('Tasks', function() {
         fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file === true);
+        assert(!!imported_file, true);
         
         // post-test cleanup
         if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -658,72 +593,30 @@ describe('Tasks', function() {
 });
 
 describe('Archive', async function() {
+    const archive_path = path.join('test', 'archives');
+    fs.ensureDirSync(archive_path);
+    const archive_file_path = path.join(archive_path, 'archive_video.txt');
+    const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
     beforeEach(async function() {
-        await db_api.connectToDB();
+        if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
-        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+        fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
+
+        if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
+        fs.writeFileSync(blacklist_file_path, '');
+    });
+    
+    it('Delete from archive', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
+        const new_archive = fs.readFileSync(archive_file_path);
+        assert(!new_archive.includes('testing2'));
     });
 
-    afterEach(async function() {
+    it('Delete from archive - blacklist', async function() {
-        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
-    });
+        const new_archive = fs.readFileSync(archive_file_path);
-
+        const new_blacklist = fs.readFileSync(blacklist_file_path);
-    it('Import archive', async function() {
+        assert(!new_archive.includes('testing2'));
-        const archive_text = `
+        assert(new_blacklist.includes('testing2'));
-            testextractor1 testing1
-            testextractor1 testing2
-            testextractor2 testing1
-            testextractor1 testing3
-
-        `;
-        const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
-        assert(count === 4)
-        const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
-        console.log(archive_items);
-        assert(archive_items.length === 4);
-        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
-        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
-
-        const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
-        assert(success);
-    });
-
-    it('Get archive', async function() {
-        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
-
-        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
-        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
-
-        assert(archive_item1 && archive_item2);
-    });
-
-    it('Archive duplicates', async function() {
-        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
-
-        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
-
-        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
-        assert(count === 3);
-    });
-
-    it('Remove from archive', async function() {
-        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
-
-        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
-        assert(success);
-
-        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
-        assert(!!archive_item1);
-        
-        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
-        assert(!archive_item2);
-
-        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
-        assert(!!archive_item3);
     });
 });
 
@@ -733,129 +626,4 @@ describe('Utils', async function() {
         const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
         assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
     });
-
-    it('Convert flat object to nested object', async function() {
-        // No modfication
-        const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
-        const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
-        assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
-        assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
-
-        // Standard setup
-        const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
-        const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
-        assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
-        assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
-
-        // Nested branches
-        const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
-        const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
-        assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
-        assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
-    });
-});
-
-describe('Categories', async function() {
-    beforeEach(async function() {
-        await db_api.connectToDB();
-        const new_category = {
-            name: 'test_category',
-            uid: uuid(),
-            rules: [],
-            custom_output: ''
-        };
-    
-        await db_api.insertRecordIntoTable('categories', new_category);
-    });
-
-    afterEach(async function() {
-        await db_api.removeAllRecords('categories', {name: 'test_category'});
-    });
-
-    it('Categorize - includes', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'includes',
-            property: 'title',
-            value: 'Sample'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        assert(category && category.name === 'test_category');
-    });
-
-    it('Categorize - not includes', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'not_includes',
-            property: 'title',
-            value: 'Sample'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        assert(!category);
-    });
-
-    it('Categorize - equals', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'equals',
-            property: 'uploader',
-            value: 'Sample Uploader'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        console.log(category);
-        assert(category && category.name === 'test_category');
-    });
-
-    it('Categorize - not equals', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'not_equals',
-            property: 'uploader',
-            value: 'Sample Uploader'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        assert(!category);
-    });
-
-    it('Categorize - AND', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'equals',
-            property: 'uploader',
-            value: 'Sample Uploader'
-        });
-
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: 'and',
-            comparator: 'not_includes',
-            property: 'title',
-            value: 'Sample'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        assert(!category);
-    });
-
-    it('Categorize - OR', async function() {
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'equals',
-            property: 'uploader',
-            value: 'Sample Uploader'
-        });
-
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: 'or',
-            comparator: 'not_includes',
-            property: 'title',
-            value: 'Sample'
-        });
-
-        const category = await categories_api.categorize([sample_video_json]);
-        assert(category);
-    });
 });

backend/twitch.js

@@ -4,29 +4,19 @@ const logger = require('./logger');
 const moment = require('moment');
 const fs = require('fs-extra')
 const path = require('path');
-const { promisify } = require('util');
-const child_process = require('child_process');
-const commandExistsSync = require('command-exists').sync;
 
-async function getCommentsForVOD(vodId) {
+async function getCommentsForVOD(clientID, clientSecret, vodId) {
+    const { promisify } = require('util');
+    const child_process = require('child_process');
     const exec = promisify(child_process.exec);
     
     // Reject invalid params to prevent command injection attack
-    if (!vodId.match(/^[0-9a-z]+$/)) {
+    if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
-        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
+        logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
         return null;
     }
 
-    const is_windows = process.platform === 'win32';
+    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
-    const cliExt = is_windows ? '.exe' : ''
-    const cliPath = `TwitchDownloaderCLI${cliExt}`
-
-    if (!commandExistsSync(cliPath)) {
-        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
-        return null;
-    }
-
-    const result = await exec(`${cliPath} chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
 
     if (result['stderr']) {
         logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -83,7 +73,9 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
 async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
     const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
     const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const chat = await getCommentsForVOD(vodId);
+    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
 
     // save file if needed params are included
     let file_path = null;

backend/utils.js

@@ -4,7 +4,6 @@ const ffmpeg = require('fluent-ffmpeg');
 const archiver = require('archiver');
 const fetch = require('node-fetch');
 const ProgressBar = require('progress');
-const winston = require('winston');
 
 const config_api = require('./config');
 const logger = require('./logger');
@@ -13,7 +12,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-exports.getTrueFileName = (unfixed_path, type) => {
+function getTrueFileName(unfixed_path, type) {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -28,13 +27,13 @@ exports.getTrueFileName = (unfixed_path, type) => {
     return fixed_path;
 }
 
-exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
+async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
     // return empty array if the path doesn't exist
     if (!(await fs.pathExists(basePath))) return [];
 
     let files = [];
     const ext = type === 'audio' ? 'mp3' : 'mp4';
-    var located_files = await exports.recFindByExt(basePath, ext);
+    var located_files = await recFindByExt(basePath, ext);
     for (let i = 0; i < located_files.length; i++) {
         let file = located_files[i];
         var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -42,33 +41,33 @@ exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false)
         var stats = await fs.stat(file);
 
         var id = file_path.substring(0, file_path.length-4);
-        var jsonobj = await exports.getJSONByType(type, id, basePath);
+        var jsonobj = await getJSONByType(type, id, basePath);
         if (!jsonobj) continue;
         if (full_metadata) {
             jsonobj['id'] = id;
             files.push(jsonobj);
             continue;
         }
-        var upload_date = exports.formatDateString(jsonobj.upload_date);
+        var upload_date = formatDateString(jsonobj.upload_date);
 
         var isaudio = type === 'audio';
-        var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
                                 stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
         files.push(file_obj);
     }
     return files;
 }
 
-exports.createContainerZipFile = async (file_name, container_file_objs) => {
+async function createContainerZipFile(file_name, container_file_objs) {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
 }
 
-exports.createZipFile = async (zip_file_path, file_paths) => {
+async function createZipFile(zip_file_path, file_paths) {
     let output = fs.createWriteStream(zip_file_path);
 
     var archive = archiver('zip', {
@@ -92,11 +91,11 @@ exports.createZipFile = async (zip_file_path, file_paths) => {
     await archive.finalize();
 
     // wait a tiny bit for the zip to reload in fs
-    await exports.wait(100);
+    await wait(100);
     return zip_file_path;
 }
 
-exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
+function getJSONMp4(name, customPath, openReadPerms = false) {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -111,7 +110,7 @@ exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
     return obj;
 }
 
-exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
+function getJSONMp3(name, customPath, openReadPerms = false) {
     var obj = null;
     if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
     var jsonPath = path.join(customPath, name + ".info.json");
@@ -128,11 +127,11 @@ exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
     return obj;
 }
 
-exports.getJSON = (file_path, type) => {
+function getJSON(file_path, type) {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
     let obj = null;
-    var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
+    var jsonPath = removeFileExtension(file_path) + '.info.json';
-    var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
+    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
     if (fs.existsSync(jsonPath))
     {
         obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -143,12 +142,12 @@ exports.getJSON = (file_path, type) => {
     return obj;
 }
 
-exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
+function getJSONByType(type, name, customPath, openReadPerms = false) {
-    return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
+    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
 }
 
-exports.getDownloadedThumbnail = (file_path) => {
+function getDownloadedThumbnail(file_path) {
-    const file_path_no_extension = exports.removeFileExtension(file_path);
+    const file_path_no_extension = removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
     let webpPath = file_path_no_extension + '.webp';
@@ -164,7 +163,7 @@ exports.getDownloadedThumbnail = (file_path) => {
         return null;
 }
 
-exports.getExpectedFileSize = (input_info_jsons) => {
+function getExpectedFileSize(input_info_jsons) {
     // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
     const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
 
@@ -187,12 +186,12 @@ exports.getExpectedFileSize = (input_info_jsons) => {
     return expected_filesize;
 }
 
-exports.fixVideoMetadataPerms = (file_path, type) => {
+function fixVideoMetadataPerms(file_path, type) {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = exports.removeFileExtension(file_path);
+    const file_path_no_extension = removeFileExtension(file_path);
 
     const files_to_fix = [
         // JSONs
@@ -209,10 +208,10 @@ exports.fixVideoMetadataPerms = (file_path, type) => {
     }
 }
 
-exports.deleteJSONFile = (file_path, type) => {
+function deleteJSONFile(file_path, type) {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
-    const file_path_no_extension = exports.removeFileExtension(file_path);
+    const file_path_no_extension = removeFileExtension(file_path);
 
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
@@ -221,7 +220,58 @@ exports.deleteJSONFile = (file_path, type) => {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-exports.durationStringToNumber = (dur_str) => {
+// archive helper functions
+
+async function removeIDFromArchive(archive_path, type, id) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
+    if (!data) {
+        logger.error('Archive could not be found.');
+        return;
+    }
+
+    let dataArray = data.split('\n'); // convert file data in an array
+    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
+    let lastIndex = -1; // let say, we have not found the keyword
+
+    for (let index=0; index<dataArray.length; index++) {
+        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+            lastIndex = index; // found a line includes a id keyword
+            break;
+        }
+    }
+
+    if (lastIndex === -1) return null;
+
+    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
+
+    // UPDATE FILE WITH NEW DATA
+    const updatedData = dataArray.join('\n');
+    await fs.writeFile(archive_file, updatedData);
+    if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
+}
+
+async function writeToBlacklist(archive_folder, type, line) {
+    let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
+    // adds newline to the beginning of the line
+    line.replace('\n', '');
+    line.replace('\r', '');
+    line = '\n' + line;
+    await fs.appendFile(blacklistPath, line);
+}
+
+async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    if (await fs.pathExists(archive_path)) {
+        const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
+        if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
+    } else {
+        logger.info(`Could not find archive file for file ${uid}. Creating...`);
+        await fs.close(await fs.open(archive_file, 'w'));
+    }
+}
+
+function durationStringToNumber(dur_str) {
     if (typeof dur_str === 'number') return dur_str;
     let num_sum = 0;
     const dur_str_parts = dur_str.split(':');
@@ -231,22 +281,23 @@ exports.durationStringToNumber = (dur_str) => {
     return num_sum;
 }
 
-exports.getMatchingCategoryFiles = (category, files) => {
+function getMatchingCategoryFiles(category, files) {
     return files && files.filter(file => file.category && file.category.uid === category.uid);
 }
 
-exports.addUIDsToCategory = (category, files) => {
+function addUIDsToCategory(category, files) {
-    const files_that_match = exports.getMatchingCategoryFiles(category, files);
+    const files_that_match = getMatchingCategoryFiles(category, files);
     category['uids'] = files_that_match.map(file => file.uid);
     return files_that_match;
 }
 
-exports.getCurrentDownloader = () => {
+function getCurrentDownloader() {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
     return details_json['downloader'];
 }
 
-exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
+async function recFindByExt(base, ext, files, result, recursive = true)
+{
     files = files || (await fs.readdir(base))
     result = result || []
 
@@ -255,7 +306,7 @@ exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
         if ( (await fs.stat(newbase)).isDirectory() )
         {
             if (!recursive) continue;
-            result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
         {
@@ -268,17 +319,17 @@ exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
     return result
 }
 
-exports.removeFileExtension = (filename) => {
+function removeFileExtension(filename) {
     const filename_parts = filename.split('.');
     filename_parts.splice(filename_parts.length - 1);
     return filename_parts.join('.');
 }
 
-exports.formatDateString = (date_string) => {
+function formatDateString(date_string) {
     return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
 }
 
-exports.createEdgeNGrams = (str) => {
+function createEdgeNGrams(str) {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
@@ -300,7 +351,7 @@ exports.createEdgeNGrams = (str) => {
 
 // ffmpeg helper functions
 
-exports.cropFile = async (file_path, start, end, ext) => {
+async function cropFile(file_path, start, end, ext) {
     return new Promise(resolve => {
         const temp_file_path = `${file_path}.cropped${ext}`;
         let base_ffmpeg_call = ffmpeg(file_path);
@@ -329,13 +380,13 @@ exports.cropFile = async (file_path, start, end, ext) => {
  * setTimeout, but its a promise.
  * @param {number} ms
  */
-exports.wait = async (ms) => {
+ async function wait(ms) {
     await new Promise(resolve => {
         setTimeout(resolve, ms);
     });
 }
 
-exports.checkExistsWithTimeout = async (filePath, timeout) => {
+async function checkExistsWithTimeout(filePath, timeout) {
     return new Promise(function (resolve, reject) {
 
         var timer = setTimeout(function () {
@@ -364,7 +415,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
 }
 
 // helper function to download file using fetch
-exports.fetchFile = async (url, path, file_label) => {
+async function fetchFile(url, path, file_label) {
     var len = null;
     const res = await fetch(url);
 
@@ -391,7 +442,7 @@ exports.fetchFile = async (url, path, file_label) => {
     });
 }
 
-exports.restartServer = async (is_update = false) => {
+async function restartServer(is_update = false) {
     logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
 
     // the following line restarts the server through pm2
@@ -404,7 +455,7 @@ exports.restartServer = async (is_update = false) => {
 //  - if already exists and doesn't have value, ignore
 //  - if it doesn't exist and has value, add both arg and value
 //  - if it doesn't exist and doesn't have value, add arg
-exports.injectArgs = (original_args, new_args) => {
+function injectArgs(original_args, new_args) {
     const updated_args = original_args.slice();
     try {
         for (let i = 0; i < new_args.length; i++) {
@@ -414,11 +465,10 @@ exports.injectArgs = (original_args, new_args) => {
             if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
                 if (original_args.includes(new_arg)) {
                     const original_index = original_args.indexOf(new_arg);
-                    updated_args.splice(original_index, 2);
+                    original_args.splice(original_index, 2);
                 }
 
                 updated_args.push(new_arg, new_args[i + 1]);
-                i++; // we need to skip the arg value on the next loop
             } else {
                 if (!original_args.includes(new_arg)) {
                     updated_args.push(new_arg);
@@ -433,11 +483,11 @@ exports.injectArgs = (original_args, new_args) => {
     return updated_args;
 }
 
-exports.filterArgs = (args, args_to_remove) => {
+function filterArgs(args, args_to_remove) {
     return args.filter(x => !args_to_remove.includes(x));
 }
 
-exports.searchObjectByString = (o, s) => {
+const searchObjectByString = function(o, s) {
     s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
     s = s.replace(/^\./, '');           // strip a leading dot
     var a = s.split('.');
@@ -452,7 +502,7 @@ exports.searchObjectByString = (o, s) => {
     return o;
 }
 
-exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
+function stripPropertiesFromObject(obj, properties, whitelist = false) {
     if (!whitelist) {
         const new_obj = JSON.parse(JSON.stringify(obj));
         for (let field of properties) {
@@ -468,7 +518,7 @@ exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
     return new_obj;
 }
 
-exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
+function getArchiveFolder(type, user_uid = null, sub = null) {
     const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
     const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
@@ -487,49 +537,6 @@ exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
     }
 }
 
-exports.getBaseURL = () => {
-    return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
-}
-
-exports.updateLoggerLevel = (new_logger_level) => {
-    const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
-    if (!possible_levels.includes(new_logger_level)) {
-        logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
-        new_logger_level = 'info';
-    }
-    logger.level = new_logger_level;
-    winston.loggers.get('console').level = new_logger_level;
-    logger.transports[2].level = new_logger_level;
-}
-
-exports.convertFlatObjectToNestedObject = (obj) => {
-    const result = {};
-    for (const key in obj) {
-      const nestedKeys = key.split('.');
-      let currentObj = result;
-      for (let i = 0; i < nestedKeys.length; i++) {
-        if (i === nestedKeys.length - 1) {
-          currentObj[nestedKeys[i]] = obj[key];
-        } else {
-          currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
-          currentObj = currentObj[nestedKeys[i]];
-        }
-      }
-    }
-    return result;
-}
-
-exports.getDirectoriesInDirectory = async (basePath) => {
-    try {
-        const files = await fs.readdir(basePath, { withFileTypes: true });
-        return files
-            .filter((file) => file.isDirectory())
-            .map((file) => path.join(basePath, file.name));
-    } catch (err) {
-        return [];
-    }
-}
-
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -547,7 +554,38 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
     this.view_count = view_count;
     this.height = height;
     this.abr = abr;
-    this.favorite = false;
+}
-}   
-exports.File = File;
 
+module.exports = {
+    getJSONMp3: getJSONMp3,
+    getJSONMp4: getJSONMp4,
+    getJSON: getJSON,
+    getTrueFileName: getTrueFileName,
+    getDownloadedThumbnail: getDownloadedThumbnail,
+    getExpectedFileSize: getExpectedFileSize,
+    fixVideoMetadataPerms: fixVideoMetadataPerms,
+    deleteJSONFile: deleteJSONFile,
+    removeIDFromArchive: removeIDFromArchive,
+    writeToBlacklist: writeToBlacklist,
+    deleteFileFromArchive: deleteFileFromArchive,
+    getDownloadedFilesByType: getDownloadedFilesByType,
+    createContainerZipFile: createContainerZipFile,
+    durationStringToNumber: durationStringToNumber,
+    getMatchingCategoryFiles: getMatchingCategoryFiles,
+    getCurrentDownloader: getCurrentDownloader,
+    recFindByExt: recFindByExt,
+    removeFileExtension: removeFileExtension,
+    formatDateString: formatDateString,
+    cropFile: cropFile,
+    createEdgeNGrams: createEdgeNGrams,
+    wait: wait,
+    checkExistsWithTimeout: checkExistsWithTimeout,
+    fetchFile: fetchFile,
+    restartServer: restartServer,
+    injectArgs: injectArgs,
+    filterArgs: filterArgs,
+    searchObjectByString: searchObjectByString,
+    stripPropertiesFromObject: stripPropertiesFromObject,
+    getArchiveFolder: getArchiveFolder,
+    File: File
+}

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.3.2"
+appVersion: "4.3"

docker-compose.yml

@@ -18,8 +18,7 @@ services:
             - "8998:17442"
         image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        # If you are using a Raspberry Pi, use mongo:4.4.18
+        image: mongo
-        image: mongo:4
         logging:
             driver: "none"          
         container_name: mongo-db

docker-utils/GetTwitchDownloader.py

@@ -1,69 +0,0 @@
-import platform
-import requests
-import shutil
-import os
-import re
-import sys
-from collections import OrderedDict
-
-from github import Github
-
-machine = platform.machine()
-
-# https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
-MACHINES_TO_ZIP = OrderedDict([
-    ("x86_64", "Linux-x64"),
-    ("aarch64", "LinuxArm64"),
-    ("armv8", "LinuxArm64"),
-    ("arm", "LinuxArm"),
-    ("AMD64", "Windows-x64")
-])
-
-def getZipName():
-    for possibleMachine, possibleZipName in MACHINES_TO_ZIP.items():
-        if possibleMachine in machine:
-            return possibleZipName
-
-def getLatestFileInRepo(repo, search_string):
-    # Create an unauthenticated instance of the Github object
-    g = Github(os.environ.get('GH_TOKEN'))
-
-    # Replace with the repository owner and name
-    repo = g.get_repo(repo)
-
-    # Get all releases of the repository
-    releases = repo.get_releases()
-
-    # Loop through the releases in reverse order (from latest to oldest)
-    for release in list(releases):
-        # Get the release assets (files attached to the release)
-        assets = release.get_assets()
-
-        # Loop through the assets
-        for asset in assets:
-            if re.search(search_string, asset.name):
-                print(f'Downloading: {asset.name}')
-                response = requests.get(asset.browser_download_url)
-                with open(asset.name, 'wb') as f:
-                    f.write(response.content)
-                print(f'Download complete: {asset.name}. Unzipping...')
-                shutil.unpack_archive(asset.name, './')
-                print(f'Unzipping complete!')
-                os.remove(asset.name)
-                break
-        else:
-            continue
-        break
-    else:
-        # If no matching release is found, print a message
-        print(f'No release found with {search_string}')
-
-def getLatestCLIRelease():
-    zipName = getZipName()
-    if not zipName:
-        print(f"GetTwitchDownloader.py could not get valid path for '{machine}'. Exiting...")
-        sys.exit(1)
-    searchString = r'.*CLI.*' + zipName
-    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
-
-getLatestCLIRelease()

docker-utils/fetch-twitchdownloader.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
-# and also optimizing some code with this commit.
-# xoxo :D
-
-case $(uname -m) in
-  x86_64)
-    ARCH=Linux-x64;;
-  aarch64)
-    ARCH=LinuxArm64;;
-  armhf)
-    ARCH=LinuxArm;;
-  armv7)
-    ARCH=LinuxArm;;
-  armv7l)
-    ARCH=LinuxArm;;
-  *)
-    echo "Unsupported architecture: $(uname -m)"
-    exit 1
-esac
-
-echo "(INFO) Architecture detected: $ARCH"
-echo "(1/5) READY - Install unzip"
-apt-get update && apt-get -y install unzip curl jq libicu70
-VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases" | jq -r --arg arch "$ARCH" '[.[] | select(.assets | length > 0) | select(.assets[].name | contains("CLI") and contains($arch))] | max_by(.published_at) | .tag_name')
-echo "(2/5) DOWNLOAD - Acquire twitchdownloader"
-curl -o twitchdownloader.zip \
-    --connect-timeout 5 \
-    --max-time 120 \
-    --retry 5 \
-    --retry-delay 0 \
-    --retry-max-time 40 \
-    -L "https://github.com/lay295/TwitchDownloader/releases/download/$VERSION/TwitchDownloaderCLI-$VERSION-$ARCH.zip"
-unzip twitchdownloader.zip
-chmod +x TwitchDownloaderCLI
-echo "(3/5) Smoke test"
-./TwitchDownloaderCLI --help
-cp ./TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI

ffmpeg-fetch.sh

@@ -26,11 +26,11 @@ apt-get update && apt-get -y install curl xz-utils
 echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
 curl -o ffmpeg.txz \
     --connect-timeout 5 \
-    --max-time 120 \
+    --max-time 10 \
     --retry 5 \
     --retry-delay 0 \
     --retry-max-time 40 \
-    "https://johnvansickle.com/ffmpeg/old-releases/ffmpeg-5.1.1-${ARCH}-static.tar.xz"
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
 echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.3.2",
+  "version": "4.3.0",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -21,60 +21,62 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^15.0.1",
+    "@angular-devkit/core": "^14.0.4",
-    "@angular/animations": "^15.0.1",
+    "@angular/animations": "^14.0.4",
-    "@angular/cdk": "^15.0.0",
+    "@angular/cdk": "^14.0.4",
-    "@angular/common": "^15.0.1",
+    "@angular/common": "^14.0.4",
-    "@angular/compiler": "^15.0.1",
+    "@angular/compiler": "^14.0.4",
-    "@angular/core": "^15.0.1",
+    "@angular/core": "^14.0.4",
-    "@angular/forms": "^15.0.1",
+    "@angular/forms": "^14.0.4",
-    "@angular/localize": "^15.0.1",
+    "@angular/localize": "^14.0.4",
-    "@angular/material": "^15.0.0",
+    "@angular/material": "^14.0.4",
-    "@angular/platform-browser": "^15.0.1",
+    "@angular/platform-browser": "^14.0.4",
-    "@angular/platform-browser-dynamic": "^15.0.1",
+    "@angular/platform-browser-dynamic": "^14.0.4",
-    "@angular/router": "^15.0.1",
+    "@angular/router": "^14.0.4",
     "@fontsource/material-icons": "^4.5.4",
-    "@ngneat/content-loader": "^7.0.0",
+    "@ngneat/content-loader": "^5.0.0",
-    "@videogular/ngx-videogular": "^6.0.0",
+    "@videogular/ngx-videogular": "^5.0.1",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
-    "filesize": "^10.0.7",
+    "filesize": "^6.1.0",
+    "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
-    "ngx-avatars": "^1.4.1",
+    "ng-lazyload-image": "^7.0.1",
-    "ngx-file-drop": "^15.0.0",
+    "ngx-avatars": "^1.3.1",
+    "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
-    "rxjs-compat": "^6.6.7",
+    "rxjs-compat": "^6.0.0-rc.0",
     "tslib": "^2.0.0",
-    "typescript": "~4.8.4",
+    "typescript": "~4.6.3",
     "xliff-to-json": "^1.0.4",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^15.0.1",
+    "@angular-devkit/build-angular": "^14.0.4",
-    "@angular/cli": "^15.0.1",
+    "@angular/cli": "^14.0.4",
-    "@angular/compiler-cli": "^15.0.1",
+    "@angular/compiler-cli": "^14.0.4",
-    "@angular/language-service": "^15.0.1",
+    "@angular/language-service": "^14.0.4",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "^4.3.1",
+    "@types/jasmine": "~3.6.0",
     "@types/node": "^12.11.1",
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
-    "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
+    "electron": "^19.0.6",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.4.2",
+    "karma": "~6.3.16",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
-    "karma-jasmine": "~5.1.0",
+    "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.23.0",
+    "openapi-typescript-codegen": "^0.21.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -3,7 +3,6 @@
 /* eslint-disable */
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
-export type { Archive } from './models/Archive';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
 export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
@@ -27,10 +26,8 @@ export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
 export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
-export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
-export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
 export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
 export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -53,8 +50,6 @@ export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
-export type { GetArchivesRequest } from './models/GetArchivesRequest';
-export type { GetArchivesResponse } from './models/GetArchivesResponse';
 export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
 export type { GetDownloadRequest } from './models/GetDownloadRequest';
 export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -68,7 +63,6 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
 export type { GetLogsResponse } from './models/GetLogsResponse';
 export type { GetMp3sResponse } from './models/GetMp3sResponse';
 export type { GetMp4sResponse } from './models/GetMp4sResponse';
-export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
 export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
 export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
 export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -79,22 +73,16 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
 export type { GetTaskRequest } from './models/GetTaskRequest';
 export type { GetTaskResponse } from './models/GetTaskResponse';
 export type { GetUsersResponse } from './models/GetUsersResponse';
-export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
 export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
 export type { inline_response_200_15 } from './models/inline_response_200_15';
 export type { LoginRequest } from './models/LoginRequest';
 export type { LoginResponse } from './models/LoginResponse';
-export type { Notification } from './models/Notification';
-export { NotificationAction } from './models/NotificationAction';
-export { NotificationType } from './models/NotificationType';
 export type { Playlist } from './models/Playlist';
 export type { RegisterRequest } from './models/RegisterRequest';
 export type { RegisterResponse } from './models/RegisterResponse';
-export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
 export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
-export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
 export type { SharingToggle } from './models/SharingToggle';
 export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
@@ -104,7 +92,6 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
 export type { SuccessObject } from './models/SuccessObject';
 export type { TableInfo } from './models/TableInfo';
 export type { Task } from './models/Task';
-export { TaskType } from './models/TaskType';
 export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
 export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
 export type { TransferDBRequest } from './models/TransferDBRequest';
@@ -121,10 +108,8 @@ export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';
 export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
-export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
 export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
 export type { UpdateUserRequest } from './models/UpdateUserRequest';
-export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
 export type { User } from './models/User';
 export { UserPermission } from './models/UserPermission';
 export type { Version } from './models/Version';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -5,4 +5,4 @@
 export type AddFileToPlaylistRequest = {
     file_uid: string;
     playlist_id: string;
-};
+};

src/api-types/models/Archive.ts

@@ -1,16 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import type { FileType } from './FileType';
-
-export type Archive = {
-    extractor: string;
-    id: string;
-    type: FileType;
-    title: string;
-    user_uid?: string;
-    sub_id?: string;
-    timestamp: number;
-    uid: string;
-};

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
 export type BaseChangePermissionsRequest = {
     permission: UserPermission;
     new_value: YesNo;
-};
+};

src/api-types/models/Category.ts

@@ -12,4 +12,4 @@ export type Category = {
      * Overrides file output for downloaded files in category
      */
     custom_output?: string;
-};
+};

src/api-types/models/CategoryRule.ts

@@ -22,4 +22,4 @@ export namespace CategoryRule {
     }
 
 
-}
+}

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
 
 export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
 role: string;
-});
+});

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
 
 export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
 user_uid: string;
-});
+});

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
      * UID of the concurrent stream
      */
     uid: string;
-};
+};

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
 
 export type CheckConcurrentStreamResponse = {
     stream: ConcurrentStream;
-};
+};

src/api-types/models/ClearDownloadsRequest.ts

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
     clear_finished?: boolean;
     clear_paused?: boolean;
     clear_errors?: boolean;
-};
+};

src/api-types/models/ConcurrentStream.ts

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
     playback_timestamp?: number;
     unix_timestamp?: number;
     playing?: boolean;
-};
+};

src/api-types/models/Config.ts

@@ -4,4 +4,4 @@
 
 export type Config = {
     YoutubeDLMaterial: any;
-};
+};

src/api-types/models/ConfigResponse.ts

@@ -7,4 +7,4 @@ import type { Config } from './Config';
 export type ConfigResponse = {
     config_file: Config;
     success: boolean;
-};
+};

src/api-types/models/CreateCategoryRequest.ts

@@ -4,4 +4,4 @@
 
 export type CreateCategoryRequest = {
     name: string;
-};
+};

src/api-types/models/CreateCategoryResponse.ts

@@ -7,4 +7,4 @@ import type { Category } from './Category';
 export type CreateCategoryResponse = {
     new_category?: Category;
     success?: boolean;
-};
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
     playlistName: string;
     uids: Array<string>;
     thumbnailURL: string;
-};
+};

src/api-types/models/CreatePlaylistResponse.ts

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
 export type CreatePlaylistResponse = {
     new_playlist: Playlist;
     success: boolean;
-};
+};

src/api-types/models/CropFileSettings.ts

@@ -5,4 +5,4 @@
 export type CropFileSettings = {
     cropFileStart: number;
     cropFileEnd: number;
-};
+};

src/api-types/models/DBBackup.ts

@@ -17,4 +17,4 @@ export namespace DBBackup {
     }
 
 
-}
+}

src/api-types/models/DBInfoResponse.ts

@@ -14,6 +14,5 @@ subscriptions?: TableInfo;
 users?: TableInfo;
 roles?: TableInfo;
 download_queue?: TableInfo;
-archives?: TableInfo;
-};
 };
+};

src/api-types/models/DatabaseFile.ts

@@ -26,7 +26,6 @@ export type DatabaseFile = {
     path: string;
     upload_date: string;
     uid: string;
-    user_uid?: string;
     sharingEnabled?: boolean;
     category?: Category;
     view_count?: number;
@@ -41,5 +40,4 @@ export type DatabaseFile = {
      * In Kbps
      */
     abr?: number;
-    favorite: boolean;
+};
-};

src/api-types/models/DeleteAllFilesResponse.ts

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
      * Number of files removed
      */
     delete_count?: number;
-};
+};

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import type { Archive } from './Archive';
-
-export type DeleteArchiveItemsRequest = {
-    archives: Array<Archive>;
-};

src/api-types/models/DeleteCategoryRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeleteCategoryRequest = {
     category_uid: string;
-};
+};

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -5,4 +5,4 @@
 export type DeleteMp3Mp4Request = {
     uid: string;
     blacklistMode?: boolean;
-};
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -1,7 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-export type DeleteNotificationRequest = {
-    uid: string;
-};

src/api-types/models/DeletePlaylistRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeletePlaylistRequest = {
     playlist_id: string;
-};
+};

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,10 +2,14 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { SubscriptionRequestData } from './SubscriptionRequestData';
+
 export type DeleteSubscriptionFileRequest = {
-    file_uid: string;
+    file: string;
+    file_uid?: string;
+    sub: SubscriptionRequestData;
     /**
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */
     deleteForever?: boolean;
-};
+};

src/api-types/models/DeleteUserRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeleteUserRequest = {
     uid: string;
-};
+};

src/api-types/models/Download.ts

@@ -19,12 +19,8 @@ export type Download = {
      * Error text, set if download fails.
      */
     error?: string | null;
-    /**
-     * Error type, may or may not be set in case of an error
-     */
-    error_type?: string | null;
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
     prefetched_info?: any;
-};
+};

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,9 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { FileType } from './FileType';
-
 export type DownloadArchiveRequest = {
-    type?: FileType;
+    sub: {
-    sub_id?: string;
+archive_dir: string;
 };
+};

src/api-types/models/DownloadFileRequest.ts

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
     playlist_id?: string;
     url?: string;
     type?: FileType;
-};
+};

src/api-types/models/DownloadRequest.ts

@@ -35,18 +35,10 @@ export type DownloadRequest = {
      * Height of the video, if known
      */
     selectedHeight?: string;
-    /**
-     * Max height that should be used, useful for playlists. selectedHeight will override this.
-     */
-    maxHeight?: string;
     /**
      * Specify ffmpeg/avconv audio quality
      */
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
-    /**
+};
-     * If using youtube-dl archive, download will ignore it
-     */
-    ignoreArchive?: boolean;
-};

src/api-types/models/DownloadResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type DownloadResponse = {
     download?: Download;
-};
+};

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
      */
     uuid?: string;
     sub?: Subscription;
-};
+};

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
 
 export type DownloadTwitchChatByVODIDResponse = {
     chat: Array<TwitchChatMessage>;
-};
+};

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -4,4 +4,4 @@
 
 export type DownloadVideosForSubscriptionRequest = {
     subID: string;
-};
+};

src/api-types/models/FileType.ts

@@ -5,4 +5,4 @@
 export enum FileType {
     AUDIO = 'audio',
     VIDEO = 'video',
-}
+}

src/api-types/models/FileTypeFilter.ts

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
     AUDIO_ONLY = 'audio_only',
     VIDEO_ONLY = 'video_only',
     BOTH = 'both',
-}
+}

src/api-types/models/GenerateArgsResponse.ts

@@ -4,4 +4,4 @@
 
 export type GenerateArgsResponse = {
     args?: Array<string>;
-};
+};

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -4,4 +4,4 @@
 
 export type GenerateNewApiKeyResponse = {
     new_api_key: string;
-};
+};

src/api-types/models/GetAllCategoriesResponse.ts

@@ -6,4 +6,4 @@ import type { Category } from './Category';
 
 export type GetAllCategoriesResponse = {
     categories: Array<Category>;
-};
+};

src/api-types/models/GetAllDownloadsRequest.ts

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
      * Filters downloads with the array
      */
     uids?: Array<string> | null;
-};
+};

src/api-types/models/GetAllDownloadsResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type GetAllDownloadsResponse = {
     downloads?: Array<Download>;
-};
+};

src/api-types/models/GetAllFilesRequest.ts

@@ -13,12 +13,8 @@ export type GetAllFilesRequest = {
      */
     text_search?: string;
     file_type_filter?: FileTypeFilter;
-    /**
-     * If set to true, only gets favorites
-     */
-    favorite_filter?: boolean;
     /**
      * Include if you want to filter by subscription
      */
     sub_id?: string;
-};
+};

src/api-types/models/GetAllFilesResponse.ts

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
      * All video playlists
      */
     playlists: Array<Playlist>;
-};
+};

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
 
 export type GetAllSubscriptionsResponse = {
     subscriptions: Array<Subscription>;
-};
+};

src/api-types/models/GetAllTasksResponse.ts

@@ -6,4 +6,4 @@ import type { Task } from './Task';
 
 export type GetAllTasksResponse = {
     tasks?: Array<Task>;
-};
+};

src/api-types/models/GetArchivesRequest.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import type { FileType } from './FileType';
-
-export type GetArchivesRequest = {
-    type?: FileType;
-    sub_id?: string;
-};

src/api-types/models/GetArchivesResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import type { Archive } from './Archive';
-
-export type GetArchivesResponse = {
-    archives: Array<Archive>;
-};

src/api-types/models/GetDBBackupsResponse.ts

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
 
 export type GetDBBackupsResponse = {
     tasks?: Array<DBBackup>;
-};
+};

src/api-types/models/GetDownloadRequest.ts

@@ -4,4 +4,4 @@
 
 export type GetDownloadRequest = {
     download_uid: string;
-};
+};

src/api-types/models/GetDownloadResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type GetDownloadResponse = {
     download?: Download;
-};
+};

src/api-types/models/GetFileFormatsRequest.ts

@@ -4,4 +4,4 @@
 
 export type GetFileFormatsRequest = {
     url?: string;
-};
+};

src/api-types/models/GetFileFormatsResponse.ts

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
     result: {
 formats?: Array<any>;
 };
-};
+};

src/api-types/models/GetFileRequest.ts

@@ -14,4 +14,4 @@ export type GetFileRequest = {
      * User UID
      */
     uuid?: string;
-};
+};

src/api-types/models/GetFileResponse.ts

@@ -7,4 +7,4 @@ import type { DatabaseFile } from './DatabaseFile';
 export type GetFileResponse = {
     success: boolean;
     file?: DatabaseFile;
-};
+};

src/api-types/models/GetFullTwitchChatRequest.ts

@@ -16,4 +16,4 @@ export type GetFullTwitchChatRequest = {
      */
     uuid?: string;
     sub?: Subscription;
-};
+};

src/api-types/models/GetFullTwitchChatResponse.ts

@@ -5,4 +5,4 @@
 export type GetFullTwitchChatResponse = {
     success: boolean;
     error?: string;
-};
+};

src/api-types/models/GetLogsRequest.ts

@@ -4,4 +4,4 @@
 
 export type GetLogsRequest = {
     lines?: number;
-};
+};