Improved tests for multi-user mode

.github/workflows/build.yml

@@ -13,11 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '14'
           cache: 'npm'
       - name: install dependencies
         run: |
@@ -33,7 +33,7 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -55,7 +55,7 @@ jobs:
           Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
       - name: upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v1
         with:
           name: youtubedl-material
           path: build
@@ -65,7 +65,7 @@ jobs:
     if: contains(github.ref, '/tags/v')
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: create release
         id: create_release
         uses: actions/create-release@v1
@@ -81,7 +81,7 @@ jobs:
           draft: true
           prerelease: false
       - name: download build artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v1
         with:
           name: youtubedl-material
           path: ${{runner.temp}}/youtubedl-material

.github/workflows/codeql-analysis.yml

@@ -30,7 +30,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v1

.github/workflows/docker-pr.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: Set hash
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
@@ -18,21 +18,10 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
-      - name: setup platform emulator
-        uses: docker/setup-qemu-action@v2
-      - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v2
-      - name: build & push images
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
-          #platforms: linux/amd64
-          push: false
-          tags: tzahi12345/youtubedl-material:nightly-pr
+      - name: Build docker images
+        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker-release.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
 
       - name: Set hash
         id: vars
@@ -27,7 +27,7 @@ jobs:
 
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -57,13 +57,13 @@ jobs:
             type=raw,value=latest
 
       - name: setup platform emulator
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v1
 
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -76,11 +76,11 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: build & push images
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64/v8
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/docker.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
 
       - name: Set hash
         id: vars
@@ -34,17 +34,17 @@ jobs:
 
       - name: create-json
         id: create-json
-        uses: jsdaniell/create-json@v1.2.2
+        uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
           json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
 
       - name: setup platform emulator
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v1
 
       - name: setup multi-arch docker build
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Generate Docker image metadata
         id: docker-meta
@@ -63,7 +63,7 @@ jobs:
             type=sha,prefix=sha-,format=short
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -76,11 +76,11 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: build & push images
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/mocha.yml

@@ -1,40 +0,0 @@
-name: Tests
-'on':
-  push:
-    branches:
-      - master
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-
-jobs:
-  test:
-    name: 'Backend - mocha'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node:
-          - 16
-    steps:
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '${{ matrix.node }}'
-      - uses: actions/checkout@v3
-      - name: 'Cache node_modules'
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-v${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-v${{ matrix.node }}-
-          working-directory: ./backend
-      - uses: FedericoCarboni/setup-ffmpeg@v2
-        id: setup-ffmpeg
-      - name: Install Dependencies
-        run: npm install
-        working-directory: ./backend
-      - name: Run All Node.js Tests
-        run: npm run test
-        working-directory: ./backend

.vscode/launch.json

@@ -4,20 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Dev: Debug Backend",
-            "request": "launch",
-            "runtimeArgs": [
-                "run-script",
-                "debug"
-            ],
-            "runtimeExecutable": "npm",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "node",
-            "cwd": "${workspaceFolder}/backend"
-        },
         {
             "type": "node",
             "request": "attach",

.vscode/settings.json

@@ -3,6 +3,6 @@
     "mochaExplorer.cwd": "backend",
     "mochaExplorer.globImplementation": "vscode",
     "mochaExplorer.env": {
-        // "YTDL_MODE": "debug"
+        "YTDL_MODE": "debug"
     }
 }

DEVELOPMENT.md

@@ -1,38 +0,0 @@
-<h1>Development</h1>
-
-- [First time...](#first-time)
-  - [Setup](#setup)
-  - [Startup](#startup)
-- [Debugging the backend (VSC)](#debugging-the-backend-vsc)
-- [Deploy changes](#deploy-changes)
-  - [Frontend](#frontend)
-  - [Backend](#backend)
-
-# First time...
-
-## Setup
-Checkout the repository and navigate to the `youtubedl-material` directory.
-```bash
-vim ./src/assets/default.json # Edit settings for your local environment. This config file is just the dev config file, if YTDL_MODE is not set to "debug", then ./backend/appdata/default.json will be used
-npm -g install pm2 # Install pm2
-npm install # Install dependencies for the frontend
-cd ./backend
-npm install # Install dependencies for the backend
-cd ..
-npm run build # Build the frontend
-```
-This step have to be done only once.
-
-## Startup
-Navigate to the `youtubedl-material/backend` directory and run `npm start`.
-
-# Debugging the backend (VSC)
-Open the `youtubedl-material` directory in Visual Studio Code and run the launch configuration `Dev: Debug Backend`.
-
-# Deploy changes
-
-## Frontend
-Navigate to the `youtubedl-material` directory and run `npm run build`. Restart the backend.
-
-## Backend
-Simply restart the backend.

Dockerfile

@@ -1,17 +1,15 @@
-# Fetching our utils
-FROM ubuntu:22.04 AS utils
+# Fetching our ffmpeg
+FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY docker-utils/*.sh .
-RUN chmod +x *.sh
+COPY docker-utils/ffmpeg-fetch.sh .
+RUN chmod +x ffmpeg-fetch.sh
 RUN sh ./ffmpeg-fetch.sh
-RUN sh ./fetch-twitchdownloader.sh
 
 
 # Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
 # Go to 20.04
-FROM ubuntu:22.04 AS base
-ARG TARGETPLATFORM
+FROM ubuntu:20.04 AS base
 ARG DEBIAN_FRONTEND=noninteractive
 ENV UID=1000
 ENV GID=1000
@@ -19,26 +17,19 @@ ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
 ENV PM2_HOME=/app/pm2
 ENV ALLOW_CONFIG_MUTATIONS=true
-
-# Use NVM to get specific node version
-ENV NODE_VERSION=16.14.2
 RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
     apt update && \
-    apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 libatomic1 && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt install -y --no-install-recommends nodejs && \
+    npm -g install npm n && \
+    n 16.14.2 && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir /usr/local/nvm
-ENV PATH="/usr/local/nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
-ENV NVM_DIR=/usr/local/nvm
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
-RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
-RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
-RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
 
 # Build frontend
-ARG BUILDPLATFORM
-FROM --platform=${BUILDPLATFORM} node:16 as frontend
+FROM base as frontend
 RUN npm install -g @angular/cli
 WORKDIR /build
 COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
@@ -46,8 +37,6 @@ COPY [ "src/", "/build/src/" ]
 RUN npm install && \
     npm run build && \
     ls -al /build/backend/public
-RUN npm uninstall -g @angular/cli
-RUN rm -rf node_modules
 
 
 # Install backend deps
@@ -58,35 +47,31 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
-#FROM base as python
-# armv7 need build from source
-#WORKDIR /app
-#COPY docker-utils/GetTwitchDownloader.py .
-#RUN apt update && \
-#    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip python3-dev build-essential libffi-dev && \
-#    apt clean && \
-#    rm -rf /var/lib/apt/lists/*
-#RUN pip install PyGithub requests
-#RUN python GetTwitchDownloader.py
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
 RUN npm install -g pm2 && \
     apt update && \
     apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
-    pip install pycryptodomex && \
-    apt remove -y --purge build-essential && \
-    apt autoremove -y --purge && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
+RUN pip install pycryptodomex
 WORKDIR /app
 # User 1000 already exist from base image
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
-COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/usr/local/bin/TwitchDownloaderCLI"]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
 COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-#COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"]
+RUN chown $UID:$GID .
 RUN chmod +x /app/fix-scripts/*.sh
 # Add some persistence data
 #VOLUME ["/app/appdata"]

Public API v1.yaml

@@ -293,48 +293,6 @@ paths:
                 $ref: '#/components/schemas/UnsubscribeResponse'
       security:
         - Auth query parameter: []
-  /api/checkSubscription:
-    post:
-      tags:
-        - subscriptions
-      summary: Run a check for videos for a subscription
-      description: Runs a subscription check
-      operationId: post-api-checksubscription
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CheckSubscriptionRequest'
-      responses:
-        '200':
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/SuccessObject'
-      security:
-        - Auth query parameter: []
-  /api/cancelCheckSubscription:
-    post:
-      tags:
-        - subscriptions
-      summary: Cancels check for videos for a subscription
-      description: Cancels subscription check
-      operationId: post-api-checksubscription
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CheckSubscriptionRequest'
-      responses:
-        '200':
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/SuccessObject'
-      security:
-        - Auth query parameter: []
   /api/deleteSubscriptionFile:
     post:
       tags:
@@ -1800,14 +1758,14 @@ components:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
       required:
         - task_key
     UpdateTaskScheduleRequest:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         new_schedule:
           $ref: '#/components/schemas/Schedule'
       required:
@@ -1817,7 +1775,7 @@ components:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         new_data:
           type: object
       required:
@@ -1827,7 +1785,7 @@ components:
       type: object
       properties:
         task_key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         new_options:
           type: object
       required:
@@ -2023,11 +1981,11 @@ components:
           type: string
     UnsubscribeRequest:
       required:
-        - sub_id
+        - sub
       type: object
       properties:
-        sub_id:
-          type: string
+        sub:
+          $ref: '#/components/schemas/SubscriptionRequestData'
         deleteMode:
           type: boolean
           description: Defaults to false
@@ -2040,13 +1998,6 @@ components:
           type: boolean
         error:
           type: string
-    CheckSubscriptionRequest:
-      required:
-        - sub_id
-      type: object
-      properties:
-        sub_id:
-          type: string
     DeleteAllFilesResponse:
       type: object
       properties:
@@ -2732,8 +2683,6 @@ components:
           type: boolean
         paused:
           type: boolean
-        cancelled:
-          type: boolean
         finished_step:
           type: boolean
         url:
@@ -2777,7 +2726,7 @@ components:
       type: object
       properties:
         key:
-          $ref: '#/components/schemas/TaskType'
+          type: string
         title:
           type: string
         last_ran:
@@ -2793,20 +2742,9 @@ components:
         error:
           type: string
         schedule:
-          $ref: '#/components/schemas/Schedule'
+          type: object
         options:
           type: object
-    TaskType:
-      type: string
-      enum:
-      - backup_local_db
-      - missing_files_check
-      - missing_db_records
-      - duplicate_files_check
-      - youtubedl_update_check
-      - delete_old_files
-      - import_legacy_archives
-      - rebuild_database
     Schedule:
       required:
         - type
@@ -2892,8 +2830,6 @@ components:
           nullable: true
         isPlaylist:
           type: boolean
-        child_process:
-          type: object
         archive:
           type: string
         timerange:
@@ -2902,10 +2838,6 @@ components:
           type: string
         custom_output:
           type: string
-        downloading:
-          type: boolean
-        paused:
-          type: boolean
         videos:
           type: array
           items:
@@ -2945,7 +2877,6 @@ components:
         - sharing
         - advanced_download
         - downloads_manager
-        - tasks_manager
     YesNo:
       type: string
       enum:

README.md

@@ -28,28 +28,13 @@ Dark mode:
 
 NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
 
-Required dependencies:
-
-* Node.js 16
-* Python
-
-Optional dependencies:
-
-* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
-* [Twitch Downloader CLI](https://github.com/lay295/TwitchDownloader) (for downloading Twitch VOD chats)
-
-<details>
-  <summary>Debian/Ubuntu</summary>
+Debian/Ubuntu:
 
 ```bash
-curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
 sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
 ```
 
-</details>
-
-<details>
-  <summary>CentOS 7</summary>
+CentOS 7:
 
 ```bash
 sudo yum install epel-release
@@ -57,11 +42,13 @@ sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfu
 sudo yum install centos-release-scl-rh
 sudo yum install rh-nodejs12
 scl enable rh-nodejs12 bash
-curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash -
 sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 ```
 
-</details>
+Optional dependencies:
+
+* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
@@ -85,9 +72,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
 
 To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
 
-Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm run build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
-
-Lastly, type `npm -g install pm2` to install pm2 globally.
+Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
 
 The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
 

backend/app.js

@@ -1,7 +1,8 @@
-const { v4: uuid } = require('uuid');
+const { uuid } = require('uuidv4');
 const fs = require('fs-extra');
 const { promisify } = require('util');
 const auth_api = require('./authentication/auth');
+const winston = require('winston');
 const path = require('path');
 const compression = require('compression');
 const multer  = require('multer');
@@ -18,7 +19,11 @@ const CONSTS = require('./consts')
 const read_last_lines = require('read-last-lines');
 const ps = require('ps-node');
 const Feed = require('feed').Feed;
-const session = require('express-session');
+
+// needed if bin/details somehow gets deleted
+if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
+
+const youtubedl = require('youtube-dl');
 
 const logger = require('./logger');
 const config_api = require('./config.js');
@@ -29,8 +34,6 @@ const categories_api = require('./categories');
 const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
 const archive_api = require('./archive');
-const files_api = require('./files');
-const notifications_api = require('./notifications');
 
 var app = express();
 
@@ -159,7 +162,6 @@ app.use(bodyParser.json());
 
 // use passport
 app.use(auth_api.passport.initialize());
-app.use(session({ secret: uuid(), resave: true, saveUninitialized: true }))
 app.use(auth_api.passport.session());
 
 // actual functions
@@ -171,10 +173,10 @@ async function checkMigrations() {
     if (!simplified_db_migration_complete) {
         logger.info('Beginning migration: 4.1->4.2+')
         let success = await simplifyDBFileStructure();
-        success = success && await files_api.addMetadataPropertyToDB('view_count');
-        success = success && await files_api.addMetadataPropertyToDB('description');
-        success = success && await files_api.addMetadataPropertyToDB('height');
-        success = success && await files_api.addMetadataPropertyToDB('abr');
+        success = success && await db_api.addMetadataPropertyToDB('view_count');
+        success = success && await db_api.addMetadataPropertyToDB('description');
+        success = success && await db_api.addMetadataPropertyToDB('height');
+        success = success && await db_api.addMetadataPropertyToDB('abr');
         // sets migration to complete
         db.set('simplified_db_migration_complete', true).write();
         if (success) { logger.info('4.1->4.2+ migration complete!'); }
@@ -531,10 +533,15 @@ async function loadConfig() {
     if (allowSubscriptions) {
         // set downloading to false
         let subscriptions = await subscriptions_api.getAllSubscriptions();
-        subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
-        subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false, child_process: null});
+        subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
         // runs initially, then runs every ${subscriptionCheckInterval} seconds
-        subscriptions_api.watchSubscriptionsInterval();
+        const watchSubscriptionsInterval = function() {
+            watchSubscriptions();
+            const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+            setTimeout(watchSubscriptionsInterval, subscriptionsCheckInterval*1000);
+        }
+
+        watchSubscriptionsInterval();
     }
 
     // start the server here
@@ -564,6 +571,62 @@ function loadConfigValues() {
     utils.updateLoggerLevel(logger_level);
 }
 
+function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
+    // frequency is once every 5 mins by default
+    const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+    let interval_in_ms = subscriptionsCheckInterval * 1000;
+    const subinterval_in_ms = interval_in_ms/subscriptions_amount;
+    return subinterval_in_ms;
+}
+
+async function watchSubscriptions() {
+    let subscriptions = await subscriptions_api.getAllSubscriptions();
+
+    if (!subscriptions) return;
+
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
+
+    let subscriptions_amount = valid_subscriptions.length;
+    let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
+
+    let current_delay = 0;
+
+    const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
+    for (let i = 0; i < valid_subscriptions.length; i++) {
+        let sub = valid_subscriptions[i];
+
+        // don't check the sub if the last check for the same subscription has not completed
+        if (subscription_timeouts[sub.id]) {
+            logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`);
+            continue;
+        }
+
+        if (!sub.name) {
+            logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
+            continue;
+        }
+
+        logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
+        setTimeout(async () => {
+            const multiUserModeChanged = config_api.getConfigItem('ytdl_multi_user_mode') !== multiUserMode;
+            if (multiUserModeChanged) {
+                logger.verbose(`Skipping subscription ${sub.name} due to multi-user mode change.`);
+                return;
+            }
+            await subscriptions_api.getVideosForSub(sub, sub.user_uid);
+            subscription_timeouts[sub.id] = false;
+        }, current_delay);
+        subscription_timeouts[sub.id] = true;
+        current_delay += delay_interval;
+        const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+        if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
+    }
+}
+
 function getOrigin() {
     return url_domain.origin;
 }
@@ -591,20 +654,36 @@ function generateEnvVarConfigItem(key) {
 
 // currently only works for single urls
 async function getUrlInfos(url) {
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, ['--dump-json']);
-    if (!parsed_output || parsed_output.length !== 1) {
-        logger.error(`Failed to retrieve available formats for url: ${url}`);
-        if (err) logger.error(err);
-        return null;
-    }
-    return parsed_output[0];
+    let startDate = Date.now();
+    let result = [];
+    return new Promise(resolve => {
+        youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => {
+            let new_date = Date.now();
+            let difference = (new_date - startDate)/1000;
+            logger.debug(`URL info retrieval delay: ${difference} seconds.`);
+            if (err) {
+                logger.error(`Error during retrieving formats for ${url}: ${err}`);
+                resolve(null);
+            }
+            let try_putput = null;
+            try {
+                try_putput = JSON.parse(output);
+                result = try_putput;
+            } catch(e) {
+                logger.error(`Failed to retrieve available formats for url: ${url}`);
+            }
+            resolve(result);
+        });
+    });
 }
 
 // youtube-dl functions
 
 async function startYoutubeDL() {
     // auto update youtube-dl
-    await youtubedl_api.checkForYoutubeDLUpdate();
+    youtubedl_api.verifyBinaryExistsLinux();
+    const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
+    if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
 }
 
 app.use(function(req, res, next) {
@@ -624,7 +703,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss') || req.path.includes('/api/telegramRequest')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -645,7 +724,7 @@ const optionalJwt = async function (req, res, next) {
         const uuid = using_body ? req.body.uuid : req.query.uuid;
         const uid = using_body ? req.body.uid : req.query.uid;
         const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
-        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
+        const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
         if (file) {
             req.can_watch = true;
             return next();
@@ -856,7 +935,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+    const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
 
     res.send({
         files: files,
@@ -1022,7 +1101,7 @@ app.post('/api/incrementViewCount', async (req, res) => {
         uuid = req.user.uid;
     }
 
-    const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
+    const file_obj = await db_api.getVideo(file_uid, uuid, sub_id);
 
     const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
     const new_view_count = current_view_count + 1;
@@ -1130,10 +1209,10 @@ app.post('/api/subscribe', optionalJwt, async (req, res) => {
 
 app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
     let deleteMode = req.body.deleteMode
-    let sub_id = req.body.sub_id;
+    let sub = req.body.sub;
     let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    let result_obj = subscriptions_api.unsubscribe(sub_id, deleteMode, user_uid);
+    let result_obj = subscriptions_api.unsubscribe(sub, deleteMode, user_uid);
     if (result_obj.success) {
         res.send({
             success: result_obj.success
@@ -1150,7 +1229,7 @@ app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
     let deleteForever = req.body.deleteForever;
     let file_uid = req.body.file_uid;
 
-    let success = await files_api.deleteFile(file_uid, deleteForever);
+    let success = await db_api.deleteFile(file_uid, deleteForever);
 
     if (success) {
         res.send({
@@ -1203,49 +1282,21 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
 });
 
 app.post('/api/downloadVideosForSubscription', optionalJwt, async (req, res) => {
-    const subID = req.body.subID;
+    let subID = req.body.subID;
+    let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    const sub = subscriptions_api.getSubscription(subID);
-    subscriptions_api.getVideosForSub(sub.id);
+    let sub = subscriptions_api.getSubscription(subID, user_uid);
+    subscriptions_api.getVideosForSub(sub, user_uid);
     res.send({
         success: true
     });
 });
 
 app.post('/api/updateSubscription', optionalJwt, async (req, res) => {
-    const updated_sub = req.body.subscription;
-
-    const success = subscriptions_api.updateSubscription(updated_sub);
-    res.send({
-        success: success
-    });
-});
-
-app.post('/api/checkSubscription', optionalJwt, async (req, res) => {
-    let sub_id = req.body.sub_id;
+    let updated_sub = req.body.subscription;
     let user_uid = req.isAuthenticated() ? req.user.uid : null;
 
-    const success = subscriptions_api.getVideosForSub(sub_id, user_uid);
-    res.send({
-        success: success
-    });
-});
-
-app.post('/api/cancelCheckSubscription', optionalJwt, async (req, res) => {
-    let sub_id = req.body.sub_id;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
-
-    const success = subscriptions_api.cancelCheckSubscription(sub_id, user_uid);
-    res.send({
-        success: success
-    });
-});
-
-app.post('/api/cancelSubscriptionCheck', optionalJwt, async (req, res) => {
-    let sub_id = req.body.sub_id;
-    let user_uid = req.isAuthenticated() ? req.user.uid : null;
-
-    const success = subscriptions_api.getVideosForSub(sub_id, user_uid);
+    let success = subscriptions_api.updateSubscription(updated_sub, user_uid);
     res.send({
         success: success
     });
@@ -1266,7 +1317,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
 
-    const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1279,13 +1330,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
     let include_file_metadata = req.body.include_file_metadata;
 
-    const playlist = await files_api.getPlaylist(playlist_id, uuid);
+    const playlist = await db_api.getPlaylist(playlist_id, uuid);
     const file_objs = [];
 
     if (playlist && include_file_metadata) {
         for (let i = 0; i < playlist['uids'].length; i++) {
             const uid = playlist['uids'][i];
-            const file_obj = await files_api.getVideo(uid, uuid);
+            const file_obj = await db_api.getVideo(uid, uuid);
             if (file_obj) file_objs.push(file_obj);
             // TODO: remove file from playlist if could not be found
         }
@@ -1323,7 +1374,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
     playlist.uids.push(file_uid);
 
-    let success = await files_api.updatePlaylist(playlist);
+    let success = await db_api.updatePlaylist(playlist);
     res.send({
         success: success
     });
@@ -1331,7 +1382,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
 
 app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
     let playlist = req.body.playlist;
-    let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
+    let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
     res.send({
         success: success
     });
@@ -1361,7 +1412,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => {
     const blacklistMode = req.body.blacklistMode;
 
     let wasDeleted = false;
-    wasDeleted = await files_api.deleteFile(uid, blacklistMode);
+    wasDeleted = await db_api.deleteFile(uid, blacklistMode);
     res.send(wasDeleted);
 });
 
@@ -1393,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
 
     for (let i = 0; i < files.length; i++) {    
         let wasDeleted = false;
-        wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode);
+        wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
         if (wasDeleted) {
             delete_count++;
         }
@@ -1419,10 +1470,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
     if (playlist_id) {
         zip_file_generated = true;
         const playlist_files_to_download = [];
-        const playlist = await files_api.getPlaylist(playlist_id, uuid);
+        const playlist = await db_api.getPlaylist(playlist_id, uuid);
         for (let i = 0; i < playlist['uids'].length; i++) {
             const playlist_file_uid = playlist['uids'][i];
-            const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
+            const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
             playlist_files_to_download.push(file_obj);
         }
 
@@ -1436,7 +1487,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
         // generate zip
         file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
     } else {
-        const file_obj = await files_api.getVideo(uid, uuid, sub_id)
+        const file_obj = await db_api.getVideo(uid, uuid, sub_id)
         file_path_to_download = file_obj.path;
     }
     if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
@@ -1583,13 +1634,12 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
 
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
-        file_obj = await files_api.getVideo(uid, uuid, sub_id);
+        file_obj = await db_api.getVideo(uid, uuid, sub_id);
         if (file_obj) file_path = file_obj['path'];
         else file_path = null;
     }
     if (!fs.existsSync(file_path)) {
         logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
-        return;
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1724,10 +1774,6 @@ app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
 app.post('/api/getTasks', optionalJwt, async (req, res) => {
     const tasks = await db_api.getRecords('tasks');
     for (let task of tasks) {
-        if (!tasks_api.TASKS[task['key']]) {
-            logger.verbose(`Task ${task['key']} does not exist!`);
-            continue;
-        }
         if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime();
     }
     res.send({tasks: tasks});
@@ -1896,10 +1942,6 @@ app.post('/api/auth/register', optionalJwt, async (req, res) => {
         res.sendStatus(409);
         return;
     }
-
-    if (!userid || !username) {
-        logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
-    }
   
     const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
   
@@ -2036,25 +2078,6 @@ app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
-app.post('/api/telegramRequest', async (req, res) => {
-    if (!req.body.message  && !req.body.message.text) {
-        logger.error('Invalid Telegram request received!');
-        res.sendStatus(400);
-        return;
-    }
-    const text = req.body.message.text;
-    const regex_exp = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)?/gi;
-    const url_regex = new RegExp(regex_exp);
-    if (text.match(url_regex)) {
-        downloader_api.createDownload(text, 'video', {}, req.query.user_uid ? req.query.user_uid : null);
-        res.sendStatus(200);
-    } else {
-        logger.error('Invalid Telegram request received! Make sure you only send a valid URL.');
-        notifications_api.sendTelegramNotification({title: 'Invalid Telegram Request', body: 'Make sure you only send a valid URL.', url: text});
-        res.sendStatus(400);
-    }
-});
-
 // rss feed
 
 app.get('/api/rss', async function (req, res) {
@@ -2073,7 +2096,7 @@ app.get('/api/rss', async function (req, res) {
     const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
     const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
 
-    const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
+    const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
 
     const feed = new Feed({
             title: 'Downloads',
@@ -2122,8 +2145,6 @@ app.use(function(req, res, next) {
 
     let index_path = path.join(__dirname, 'public', 'index.html');
 
-    res.setHeader('Content-Type', 'text/html');
-
     fs.createReadStream(index_path).pipe(res);
 
 });

backend/appdata/default.json

@@ -49,10 +49,8 @@
       "use_telegram_API": false,
       "telegram_bot_token": "",
       "telegram_chat_id": "",
-      "telegram_webhook_proxy": "",
       "webhook_URL": "",
-      "discord_webhook_URL": "",
-      "slack_webhook_URL": ""
+      "discord_webhook_URL": ""
     },
     "Themes": {
       "default_theme": "default",

backend/archive.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const fs = require('fs-extra');
-const { v4: uuid } = require('uuid');
+const { uuid } = require('uuidv4');
 
 const db_api = require('./db');
 

backend/authentication/auth.js

@@ -1,10 +1,10 @@
 const config_api = require('../config');
-const CONSTS = require('../consts');
+const consts = require('../consts');
 const logger = require('../logger');
 const db_api = require('../db');
 
 const jwt = require('jsonwebtoken');
-const { v4: uuid } = require('uuid');
+const { uuid } = require('uuidv4');
 const bcrypt = require('bcryptjs');
 const fs = require('fs-extra');
 const path = require('path');
@@ -68,7 +68,14 @@ exports.initialize = function () {
 const setupRoles = async () => {
   const required_roles = {
     admin: {
-        permissions: CONSTS.AVAILABLE_PERMISSIONS
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
     },
     user: {
         permissions: [
@@ -221,7 +228,7 @@ exports.returnAuthResponse = async function(req, res) {
     user: req.user,
     token: req.token,
     permissions: await exports.userPermissions(req.user.uid),
-    available_permissions: CONSTS.AVAILABLE_PERMISSIONS
+    available_permissions: consts['AVAILABLE_PERMISSIONS']
   });
 }
 
@@ -392,8 +399,8 @@ exports.userPermissions = async function(user_uid) {
   const role_obj = await db_api.getRecord('roles', {key: role});
   const role_permissions = role_obj['permissions'];
 
-  for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
-    let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
+  for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
+    let permission = consts['AVAILABLE_PERMISSIONS'][i];
 
     const user_has_explicit_permission = user_obj['permissions'].includes(permission);
     const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

backend/categories.js

@@ -32,8 +32,10 @@ async function categorize(file_jsons) {
         return null;
     }
 
-    for (const file_json of file_jsons) {
-        for (const category of categories) {
+    for (let i = 0; i < file_jsons.length; i++) {
+        const file_json = file_jsons[i];
+        for (let j = 0; j < categories.length; j++) {
+            const category = categories[j];
             const rules = category['rules'];
     
             // if rules for current category apply, then that is the selected category

backend/config.js

@@ -1,26 +1,22 @@
 const logger = require('./logger');
 
 const fs = require('fs');
-const { BehaviorSubject } = require('rxjs');
-
-exports.CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
-exports.descriptors = {}; // to get rid of file locks when needed, TODO: move to youtube-dl.js
 
+let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
-exports.config_updated = new BehaviorSubject();
 
-exports.initialize = () => {
+function initialize() {
     ensureConfigFileExists();
     ensureConfigItemsExist();
 }
 
 function ensureConfigItemsExist() {
-    const config_keys = Object.keys(exports.CONFIG_ITEMS);
+    const config_keys = Object.keys(CONFIG_ITEMS);
     for (let i = 0; i < config_keys.length; i++) {
         const config_key = config_keys[i];
-        exports.getConfigItem(config_key);
+        getConfigItem(config_key);
     }
 }
 
@@ -61,17 +57,17 @@ function getElementNameInConfig(path) {
 /**
  * Check if config exists. If not, write default config to config path
  */
-exports.configExistsCheck = () => {
+function configExistsCheck() {
     let exists = fs.existsSync(configPath);
     if (!exists) {
-        exports.setConfigFile(DEFAULT_CONFIG);
+        setConfigFile(DEFAULT_CONFIG);
     }
 }
 
 /*
 * Gets config file and returns as a json
 */
-exports.getConfigFile = () => {
+function getConfigFile() {
     try {
         let raw_data = fs.readFileSync(configPath);
         let parsed_data = JSON.parse(raw_data);
@@ -82,40 +78,35 @@ exports.getConfigFile = () => {
     }
 }
 
-exports.setConfigFile = (config) => {
+function setConfigFile(config) {
     try {
-        const old_config = exports.getConfigFile();
         fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
-        const changes = exports.findChangedConfigItems(old_config, config);
-        if (changes.length > 0) {
-            for (const change of changes) exports.config_updated.next(change);
-        }
         return true;
     } catch(e) {
         return false;
     }
 }
 
-exports.getConfigItem = (key) => {
-    let config_json = exports.getConfigFile();
-    if (!exports.CONFIG_ITEMS[key]) {
+function getConfigItem(key) {
+    let config_json = getConfigFile();
+    if (!CONFIG_ITEMS[key]) {
         logger.error(`Config item with key '${key}' is not recognized.`);
         return null;
     }
-    let path = exports.CONFIG_ITEMS[key]['path'];
+    let path = CONFIG_ITEMS[key]['path'];
     const val = Object.byString(config_json, path);
     if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
         logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
-        exports.setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
+        setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
         return Object.byString(DEFAULT_CONFIG, path);
     }
     return Object.byString(config_json, path);
 }
 
-exports.setConfigItem = (key, value) => {
+function setConfigItem(key, value) {
     let success = false;
-    let config_json = exports.getConfigFile();
-    let path = exports.CONFIG_ITEMS[key]['path'];
+    let config_json = getConfigFile();
+    let path = CONFIG_ITEMS[key]['path'];
     let element_name = getElementNameInConfig(path);
     let parent_path = getParentPath(path);
     let parent_object = Object.byString(config_json, parent_path);
@@ -127,18 +118,20 @@ exports.setConfigItem = (key, value) => {
         parent_parent_object[parent_parent_single_key] = {};
         parent_object = Object.byString(config_json, parent_path);
     }
-    if (value === 'false') value = false;
-    if (value === 'true') value = true;
-    parent_object[element_name] = value;
 
-    success = exports.setConfigFile(config_json);
+    if (value === 'false' || value === 'true') {
+        parent_object[element_name] = (value === 'true');
+    } else {
+        parent_object[element_name] = value;
+    }
+    success = setConfigFile(config_json);
 
     return success;
 }
 
-exports.setConfigItems = (items) => {
+function setConfigItems(items) {
     let success = false;
-    let config_json = exports.getConfigFile();
+    let config_json = getConfigFile();
     for (let i = 0; i < items.length; i++) {
         let key = items[i].key;
         let value = items[i].value;
@@ -148,7 +141,7 @@ exports.setConfigItems = (items) => {
             value = (value === 'true');
         }
 
-        let item_path = exports.CONFIG_ITEMS[key]['path'];
+        let item_path = CONFIG_ITEMS[key]['path'];
         let item_parent_path = getParentPath(item_path);
         let item_element_name = getElementNameInConfig(item_path);
 
@@ -156,41 +149,28 @@ exports.setConfigItems = (items) => {
         item_parent_object[item_element_name] = value;
     }
 
-    success = exports.setConfigFile(config_json);
+    success = setConfigFile(config_json);
     return success;
 }
 
-exports.globalArgsRequiresSafeDownload = () => {
-    const globalArgs = exports.getConfigItem('ytdl_custom_args').split(',,');
+function globalArgsRequiresSafeDownload() {
+    const globalArgs = getConfigItem('ytdl_custom_args').split(',,');
     const argsThatRequireSafeDownload = ['--write-sub', '--write-srt', '--proxy'];
     const failedArgs = globalArgs.filter(arg => argsThatRequireSafeDownload.includes(arg));
     return failedArgs && failedArgs.length > 0;
 }
 
-exports.findChangedConfigItems = (old_config, new_config, path = '', changedConfigItems = [], depth = 0) => {
-    if (typeof old_config === 'object' && typeof new_config === 'object' && depth < 3) {
-        for (const key in old_config) {
-            if (Object.prototype.hasOwnProperty.call(new_config, key)) {
-                exports.findChangedConfigItems(old_config[key], new_config[key], `${path}${path ? '.' : ''}${key}`, changedConfigItems, depth + 1);
-            }
-        }
-    } else {
-        if (JSON.stringify(old_config) !== JSON.stringify(new_config)) {
-            const key = getConfigItemKeyByPath(path);
-            changedConfigItems.push({
-                key: key ? key : path.split('.')[path.split('.').length - 1], // return key in CONFIG_ITEMS or the object key
-                old_value: JSON.parse(JSON.stringify(old_config)),
-                new_value: JSON.parse(JSON.stringify(new_config))
-            });
-        }
-    }
-    return changedConfigItems;
-}
-
-function getConfigItemKeyByPath(path) {
-    const found_item = Object.values(exports.CONFIG_ITEMS).find(item => item.path === path);
-    if (found_item) return found_item['key'];
-    else return null;
+module.exports = {
+    getConfigItem: getConfigItem,
+    setConfigItem: setConfigItem,
+    setConfigItems: setConfigItems,
+    getConfigFile: getConfigFile,
+    setConfigFile: setConfigFile,
+    configExistsCheck: configExistsCheck,
+    CONFIG_ITEMS: CONFIG_ITEMS,
+    initialize: initialize,
+    descriptors: {},
+    globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
 }
 
 const DEFAULT_CONFIG = {
@@ -239,10 +219,8 @@ const DEFAULT_CONFIG = {
         "use_telegram_API": false,
         "telegram_bot_token": "",
         "telegram_chat_id": "",
-        "telegram_webhook_proxy": "",
         "webhook_URL": "",
-        "discord_webhook_URL": "",
-        "slack_webhook_URL": "",
+        "discord_webhook_URL": ""
       },
       "Themes": {
         "default_theme": "default",

backend/consts.js

@@ -154,10 +154,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_telegram_chat_id',
         'path': 'YoutubeDLMaterial.API.telegram_chat_id'
     },
-    'ytdl_telegram_webhook_proxy': {
-        'key': 'ytdl_telegram_webhook_proxy',
-        'path': 'YoutubeDLMaterial.API.telegram_webhook_proxy'
-    },
     'ytdl_webhook_url': {
         'key': 'ytdl_webhook_url',
         'path': 'YoutubeDLMaterial.API.webhook_URL'
@@ -166,10 +162,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_discord_webhook_url',
         'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
     },
-    'ytdl_slack_webhook_url': {
-        'key': 'ytdl_slack_webhook_url',
-        'path': 'YoutubeDLMaterial.API.slack_webhook_URL'
-    },
 
 
     // Themes
@@ -273,8 +265,7 @@ exports.AVAILABLE_PERMISSIONS = [
     'tasks_manager'
 ];
 
-exports.DETAILS_BIN_PATH = 'appdata/youtube-dl.json'
-exports.OUTDATED_YOUTUBEDL_VERSION = "2020.00.00";
+exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
 
 // args that have a value after it (e.g. -o <output> or -f <format>)
 const YTDL_ARGS_WITH_VALUES = [
@@ -352,11 +343,9 @@ const YTDL_ARGS_WITH_VALUES = [
     '--convert-subs'
 ];
 
-exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
-
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
 exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
 
-exports.CURRENT_VERSION = 'v4.3.2';
+exports.CURRENT_VERSION = 'v4.3.1';

backend/db.js

@@ -1,17 +1,19 @@
-const fs = require('fs-extra')
-const path = require('path')
+var fs = require('fs-extra')
+var path = require('path')
 const { MongoClient } = require("mongodb");
+const { uuid } = require('uuidv4');
 const _ = require('lodash');
 
 const config_api = require('./config');
-const utils = require('./utils')
+var utils = require('./utils')
 const logger = require('./logger');
 
 const low = require('lowdb')
 const FileSync = require('lowdb/adapters/FileSync');
 const { BehaviorSubject } = require('rxjs');
+const local_adapter = new FileSync('./appdata/local_db.json');
+const local_db = low(local_adapter);
 
-let local_db = null;
 let database = null;
 exports.database_initialized = false;
 exports.database_initialized_bs = new BehaviorSubject(false);
@@ -71,6 +73,10 @@ const tables = {
 
 const tables_list = Object.keys(tables);
 
+const local_db_defaults = {}
+tables_list.forEach(table => {local_db_defaults[table] = []});
+local_db.defaults(local_db_defaults).write();
+
 let using_local_db = null; 
 
 function setDB(input_db, input_users_db) {
@@ -79,18 +85,11 @@ function setDB(input_db, input_users_db) {
     exports.users_db = input_users_db
 }
 
-exports.initialize = (input_db, input_users_db, db_name = 'local_db.json') => {
+exports.initialize = (input_db, input_users_db) => {
     setDB(input_db, input_users_db);
 
     // must be done here to prevent getConfigItem from being called before init
     using_local_db = config_api.getConfigItem('ytdl_use_local_db');
-
-    const local_adapter = new FileSync(`./appdata/${db_name}`);
-    local_db = low(local_adapter);
-
-    const local_db_defaults = {}
-    tables_list.forEach(table => {local_db_defaults[table] = []});
-    local_db.defaults(local_db_defaults).write();
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
@@ -168,9 +167,82 @@ exports._connectToDB = async (custom_connection_string = null) => {
     }
 }
 
-exports.setVideoProperty = async (file_uid, assignment_obj) => {
-    // TODO: check if video exists, throw error if not
-    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
+exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
+    if (!file_object) file_object = generateFileObject(file_path, type);
+    if (!file_object) {
+        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
+        return false;
+    }
+
+    utils.fixVideoMetadataPerms(file_path, type);
+
+    // add thumbnail path
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
+
+    // if category exists, only include essential info
+    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
+
+    // modify duration
+    if (cropFileSettings) {
+        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
+    }
+
+    if (user_uid) file_object['user_uid'] = user_uid;
+    if (sub_id) file_object['sub_id'] = sub_id;
+
+    const file_obj = await registerFileDBManual(file_object);
+
+    // remove metadata JSON if needed
+    if (!config_api.getConfigItem('ytdl_include_metadata')) {
+        utils.deleteJSONFile(file_path, type)
+    }
+
+    return file_obj;
+}
+
+async function registerFileDBManual(file_object) {
+    // add additional info
+    file_object['uid'] = uuid();
+    file_object['registered'] = Date.now();
+    path_object = path.parse(file_object['path']);
+    file_object['path'] = path.format(path_object);
+
+    await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+
+    return file_object;
+}
+
+function generateFileObject(file_path, type) {
+    var jsonobj = utils.getJSON(file_path, type);
+    if (!jsonobj) {
+        return null;
+    } else if (!jsonobj['_filename']) {
+        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
+        return null;
+    }
+    const ext = (type === 'audio') ? '.mp3' : '.mp4'
+    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
+    // console.
+    var stats = fs.statSync(true_file_path);
+
+    const file_id = utils.removeFileExtension(path.basename(file_path));
+    var title = jsonobj.title;
+    var url = jsonobj.webpage_url;
+    var uploader = jsonobj.uploader;
+    var upload_date = utils.formatDateString(jsonobj.upload_date);
+
+    var size = stats.size;
+
+    var thumbnail = jsonobj.thumbnail;
+    var duration = jsonobj.duration;
+    var isaudio = type === 'audio';
+    var description = jsonobj.description;
+    var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
+    return file_obj;
+}
+
+function getAppendedBasePathSub(sub, base_path) {
+    return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
 exports.getFileDirectoriesAndDBs = async () => {
@@ -245,6 +317,277 @@ exports.getFileDirectoriesAndDBs = async () => {
     return dirs_to_check;
 }
 
+exports.importUnregisteredFiles = async () => {
+    const imported_files = [];
+    const dirs_to_check = await exports.getFileDirectoriesAndDBs();
+
+    // run through check list and check each file to see if it's missing from the db
+    for (let i = 0; i < dirs_to_check.length; i++) {
+        const dir_to_check = dirs_to_check[i];
+        // recursively get all files in dir's path
+        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
+
+        for (let j = 0; j < files.length; j++) {
+            const file = files[j];
+
+            // check if file exists in db, if not add it
+            const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
+            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
+            if (!file_is_registered) {
+                // add additional info
+                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
+                if (file_obj) {
+                    imported_files.push(file_obj['uid']);
+                    logger.verbose(`Added discovered file to the database: ${file.id}`);
+                } else {
+                    logger.error(`Failed to import ${file['path']} automatically.`);
+                }
+            }
+        }
+    }
+    return imported_files;
+}
+
+exports.addMetadataPropertyToDB = async (property_key) => {
+    try {
+        const dirs_to_check = await exports.getFileDirectoriesAndDBs();
+        const update_obj = {};
+        for (let i = 0; i < dirs_to_check.length; i++) {
+            const dir_to_check = dirs_to_check[i];
+
+            // recursively get all files in dir's path
+            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (file[property_key]) {
+                    update_obj[file.uid] = {[property_key]: file[property_key]};
+                }
+            }
+        }
+
+        return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
+    } catch(err) {
+        logger.error(err);
+        return false;
+    }
+}
+
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
+    const first_video = await exports.getVideo(uids[0]);
+    const thumbnailToUse = first_video['thumbnailURL'];
+    
+    let new_playlist = {
+        name: playlist_name,
+        uids: uids,
+        id: uuid(),
+        thumbnailURL: thumbnailToUse,
+        registered: Date.now(),
+        randomize_order: false
+    };
+
+    new_playlist.user_uid = user_uid ? user_uid : undefined;
+
+    await exports.insertRecordIntoTable('playlists', new_playlist);
+    
+    const duration = await exports.calculatePlaylistDuration(new_playlist);
+    await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
+
+    return new_playlist;
+}
+
+exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
+    let playlist = await exports.getRecord('playlists', {id: playlist_id});
+
+    if (!playlist) {
+        playlist = await exports.getRecord('categories', {uid: playlist_id});
+        if (playlist) {
+            const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
+            playlist['uids'] = uids;
+            playlist['auto'] = true;
+        }
+    }
+
+    // converts playlists to new UID-based schema
+    if (playlist && playlist['fileNames'] && !playlist['uids']) {
+        playlist['uids'] = [];
+        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
+        for (let i = 0; i < playlist['fileNames'].length; i++) {
+            const fileName = playlist['fileNames'][i];
+            const uid = await exports.getVideoUIDByID(fileName, user_uid);
+            if (uid) playlist['uids'].push(uid);
+            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
+        }
+        exports.updatePlaylist(playlist, user_uid);
+    }
+
+    // prevent unauthorized users from accessing the file info
+    if (require_sharing && !playlist['sharingEnabled']) return null;
+
+    return playlist;
+}
+
+exports.updatePlaylist = async (playlist) => {
+    let playlistID = playlist.id;
+
+    const duration = await exports.calculatePlaylistDuration(playlist);
+    playlist.duration = duration;
+
+    return await exports.updateRecord('playlists', {id: playlistID}, playlist);
+}
+
+exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
+    let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
+
+    if (!success) {
+        success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
+    }
+
+    if (!success) {
+        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
+    }
+
+    return success;
+}
+
+exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
+    if (!playlist_file_objs) {
+        playlist_file_objs = [];
+        for (let i = 0; i < playlist['uids'].length; i++) {
+            const uid = playlist['uids'][i];
+            const file_obj = await exports.getVideo(uid);
+            if (file_obj) playlist_file_objs.push(file_obj);
+        }
+    }
+
+    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
+}
+
+exports.deleteFile = async (uid, blacklistMode = false) => {
+    const file_obj = await exports.getVideo(uid);
+    const type = file_obj.isAudio ? 'audio' : 'video';
+    const folderPath = path.dirname(file_obj.path);
+    const ext = type === 'audio' ? 'mp3' : 'mp4';
+    const name = file_obj.id;
+    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
+
+    var jsonPath = `${file_obj.path}.info.json`;
+    var altJSONPath = `${filePathNoExtension}.info.json`;
+    var thumbnailPath = `${filePathNoExtension}.webp`;
+    var altThumbnailPath = `${filePathNoExtension}.jpg`;
+
+    jsonPath = path.join(__dirname, jsonPath);
+    altJSONPath = path.join(__dirname, altJSONPath);
+
+    let jsonExists = await fs.pathExists(jsonPath);
+    let thumbnailExists = await fs.pathExists(thumbnailPath);
+
+    if (!jsonExists) {
+        if (await fs.pathExists(altJSONPath)) {
+            jsonExists = true;
+            jsonPath = altJSONPath;
+        }
+    }
+
+    if (!thumbnailExists) {
+        if (await fs.pathExists(altThumbnailPath)) {
+            thumbnailExists = true;
+            thumbnailPath = altThumbnailPath;
+        }
+    }
+
+    let fileExists = await fs.pathExists(file_obj.path);
+
+    if (config_api.descriptors[uid]) {
+        try {
+            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
+                config_api.descriptors[uid][i].destroy();
+            }
+        } catch(e) {
+
+        }
+    }
+
+    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        // get id/extractor from JSON
+
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedID = null;
+        let retrievedExtractor = null;
+        if (info_json) {
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
+
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        if (!blacklistMode) {
+            // workaround until a files_api is created (using archive_api would make a circular dependency)
+            await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
+            // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
+        }
+    }
+
+    if (jsonExists) await fs.unlink(jsonPath);
+    if (thumbnailExists) await fs.unlink(thumbnailPath);
+
+    await exports.removeRecord('files', {uid: uid});
+
+    if (fileExists) {
+        await fs.unlink(file_obj.path);
+        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
+            return false;
+        } else {
+            return true;
+        }
+    } else {
+        // TODO: tell user that the file didn't exist
+        return true;
+    }
+}
+
+// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
+exports.getVideoUIDByID = async (file_id, uuid = null) => {
+    const file_obj = await exports.getRecord('files', {id: file_id});
+    return file_obj ? file_obj['uid'] : null;
+}
+
+exports.getVideo = async (file_uid) => {
+    return await exports.getRecord('files', {uid: file_uid});
+}
+
+exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+
+    if (favorite_filter) {
+        filter_obj['favorite'] = true;
+    }
+
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
+    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
+    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
+    
+    const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
+    const file_count = await exports.getRecords('files', filter_obj, true);
+
+    return {files, file_count};
+}
+
+exports.setVideoProperty = async (file_uid, assignment_obj) => {
+    // TODO: check if video exists, throw error if not
+    await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
+}
+
 // Basic DB functions
 
 // Create

backend/downloader.js

@@ -1,31 +1,29 @@
 const fs = require('fs-extra');
-const { v4: uuid } = require('uuid');
+const { uuid } = require('uuidv4');
 const path = require('path');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;
 
+const youtubedl = require('youtube-dl');
+
 const logger = require('./logger');
-const youtubedl_api = require('./youtube-dl');
 const config_api = require('./config');
 const twitch_api = require('./twitch');
 const { create } = require('xmlbuilder2');
 const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
-const files_api = require('./files');
 const notifications_api = require('./notifications');
 const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const download_to_child_process = {};
-
 if (db_api.database_initialized) {
-    exports.setupDownloads();
+    setupDownloads();
 } else {
     db_api.database_initialized_bs.subscribe(init => {
-        if (init) exports.setupDownloads();
+        if (init) setupDownloads();
     });
 }
 
@@ -48,7 +46,7 @@ We use checkDownloads() to move downloads through the steps and call their respe
 
 */
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null, paused = false) => {
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -61,7 +59,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             options: options,
             uid: uuid(),
             step_index: 0,
-            paused: paused,
+            paused: false,
             running: false,
             finished_step: true,
             error: null,
@@ -84,11 +82,8 @@ exports.pauseDownload = async (download_uid) => {
     } else if (download['finished']) {
         logger.info(`Download ${download_uid} could not be paused before completing.`);
         return false;
-    } else {
-        logger.info(`Pausing download ${download_uid}`);
     }
 
-    killActiveDownload(download);
     return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
 }
 
@@ -123,28 +118,21 @@ exports.cancelDownload = async (download_uid) => {
     } else if (download['finished']) {
         logger.info(`Download ${download_uid} could not be cancelled before completing.`);
         return false;
-    } else {
-        logger.info(`Cancelling download ${download_uid}`);
     }
-
-    killActiveDownload(download);
-    await handleDownloadError(download_uid, 'Cancelled', 'cancelled');
-    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true});
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
 }
 
 exports.clearDownload = async (download_uid) => {
     return await db_api.removeRecord('download_queue', {uid: download_uid});
 }
 
-async function handleDownloadError(download_uid, error_message, error_type = null) {
-    if (!download_uid) return;
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (!download || download['error']) return;
+async function handleDownloadError(download, error_message, error_type = null) {
+    if (!download || !download['uid']) return;
     notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
     await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
 }
 
-exports.setupDownloads = async () => {
+async function setupDownloads() {
     await fixDownloadState();
     setInterval(checkDownloads, 1000);
 }
@@ -190,30 +178,22 @@ async function checkDownloads() {
             if (waiting_download['sub_id']) {
                 const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
                 if (sub_missing) {
-                    handleDownloadError(waiting_download['uid'], `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
+                    handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
                     continue;
                 }
             }
             // move to next step
             running_downloads_count++;
             if (waiting_download['step_index'] === 0) {
-                exports.collectInfo(waiting_download['uid']);
+                collectInfo(waiting_download['uid']);
             } else if (waiting_download['step_index'] === 1) {
-                exports.downloadQueuedFile(waiting_download['uid']);
+                downloadQueuedFile(waiting_download['uid']);
             }
         }
     }
 }
 
-function killActiveDownload(download) {
-    const child_process = download_to_child_process[download['uid']];
-    if (download['step_index'] === 2 && child_process) {
-        youtubedl_api.killYoutubeDLProcess(child_process);
-        delete download_to_child_process[download['uid']];
-    }
-}
-
-exports.collectInfo = async (download_uid) => {
+async function collectInfo(download_uid) {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     if (download['paused']) {
         return;
@@ -236,21 +216,20 @@ exports.collectInfo = async (download_uid) => {
     // get video info prior to download
     let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
-    if (!info || info.length === 0) {
+    if (!info) {
         // info failed, error presumably already recorded
         return;
     }
 
-    // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
     const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-    if (useYoutubeDLArchive && !options.ignoreArchive && info.length === 1) {
-        const info_obj = info[0];
-        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info_obj['id'], type, download['user_uid'], download['sub_id']);
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
         if (exists_in_archive) {
-            const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
             logger.warn(error);
             if (download_uid) {
-                await handleDownloadError(download_uid, error, 'exists_in_archive');
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error, 'exists_in_archive');
                 return;
             }
         }
@@ -259,7 +238,7 @@ exports.collectInfo = async (download_uid) => {
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
-    if (info.length === 1 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
+    if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
 
     // set custom output if the category has one and re-retrieve info so the download manager has the right file name
     if (category && category['custom_output']) {
@@ -278,22 +257,26 @@ exports.collectInfo = async (download_uid) => {
     const files_to_check_for_progress = [];
 
     // store info in download for future use
-    for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
+    if (Array.isArray(info)) {
+        for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
+    } else {
+        files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
+    }
 
-    const title = info.length > 1 ? info[0]['playlist_title'] || info[0]['playlist'] : info[0]['title'];
+    const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
     await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
                                                                     finished_step: true,
                                                                     running: false,
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: title,
+                                                                    title: playlist_title ? playlist_title : info['title'],
                                                                     category: stripped_category,
                                                                     prefetched_info: null
                                                                 });
 }
 
-exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) => {
+async function downloadQueuedFile(download_uid) {
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
     if (download['paused']) {
         return;
@@ -321,112 +304,122 @@ exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) =
         const start_time = Date.now();
 
         const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
-        const file_objs = [];
+
         // download file
-        let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, customDownloadHandler);
-        if (child_process) download_to_child_process[download['uid']] = child_process;
-        const {parsed_output, err} = await callback;
-        clearInterval(download_checker);
-        let end_time = Date.now();
-        let difference = (end_time - start_time)/1000;
-        logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
-        if (!parsed_output) {
-            const errored_download = await db_api.getRecord('download_queue', {uid: download_uid});
-            if (errored_download && errored_download['paused']) return;
-            logger.error(err.toString());
-            await handleDownloadError(download_uid, err.toString(), 'unknown_error');
-            resolve(false);
-            return;
-        } else if (parsed_output) {
-            if (parsed_output.length === 0 || parsed_output[0].length === 0) {
-                // ERROR!
-                const error_message = `No output received for video download, check if it exists in your archive.`;
-                await handleDownloadError(download_uid, error_message, 'no_output');
-                logger.warn(error_message);
+        youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
+            const file_objs = [];
+            let end_time = Date.now();
+            let difference = (end_time - start_time)/1000;
+            logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
+            clearInterval(download_checker);
+            if (err) {
+                logger.error(err.stderr);
+                await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
-            }
-
-            for (const output_json of parsed_output) {
-                if (!output_json) {
-                    continue;
+            } else if (output) {
+                if (output.length === 0 || output[0].length === 0) {
+                    // ERROR!
+                    const error_message = `No output received for video download, check if it exists in your archive.`;
+                    await handleDownloadError(download, error_message, 'no_output');
+                    logger.warn(error_message);
+                    resolve(false);
+                    return;
                 }
 
-                // get filepath with no extension
-                const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
-
-                const ext = type === 'audio' ? '.mp3' : '.mp4';
-                var full_file_path = filepath_no_extension + ext;
-                var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
-
-                if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                    && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
-                        let vodId = url.split('twitch.tv/videos/')[1];
-                        vodId = vodId.split('?')[0];
-                        twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
-                }
-
-                // renames file if necessary due to bug
-                if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
                     try {
-                        fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
-                        logger.info('Renamed ' + file_name + '.webm to ' + file_name);
+                        // we have to do this because sometimes there will be leading characters before the actual json
+                        const start_idx = output[i].indexOf('{"');
+                        const clean_output = output[i].slice(start_idx, output[i].length);
+                        output_json = JSON.parse(clean_output);
                     } catch(e) {
-                        logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
+                        output_json = null;
                     }
-                }
 
-                if (type === 'audio') {
-                    let tags = {
-                        title: output_json['title'],
-                        artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
+                    if (!output_json) {
+                        continue;
                     }
-                    let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
-                    if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
+
+                    // get filepath with no extension
+                    const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
+
+                    const ext = type === 'audio' ? '.mp3' : '.mp4';
+                    var full_file_path = filepath_no_extension + ext;
+                    var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
+
+                    if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                            let vodId = url.split('twitch.tv/videos/')[1];
+                            vodId = vodId.split('?')[0];
+                            twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
+                    }
+
+                    // renames file if necessary due to bug
+                    if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
+                        try {
+                            fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
+                            logger.info('Renamed ' + file_name + '.webm to ' + file_name);
+                        } catch(e) {
+                            logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
+                        }
+                    }
+
+                    if (type === 'audio') {
+                        let tags = {
+                            title: output_json['title'],
+                            artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
+                        }
+                        let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
+                        if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
+                    }
+
+                    if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
+                        exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
+                    }
+
+                    if (options.cropFileSettings) {
+                        await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
+                    }
+
+                    // registers file in DB
+                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+
+                    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
+
+                    notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
+
+                    file_objs.push(file_obj);
                 }
 
-                if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
-                    exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
+                let container = null;
+
+                if (file_objs.length > 1) {
+                    // create playlist
+                    const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
+                } else if (file_objs.length === 1) {
+                    container = file_objs[0];
+                } else {
+                    const error_message = 'Downloaded file failed to result in metadata object.';
+                    logger.error(error_message);
+                    await handleDownloadError(download, error_message, 'no_metadata');
                 }
 
-                if (options.cropFileSettings) {
-                    await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
-                }
-
-                // registers file in DB
-                const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
-
-                await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
-
-                notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
-
-                file_objs.push(file_obj);
+                const file_uids = file_objs.map(file_obj => file_obj.uid);
+                await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
+                resolve();
             }
-
-            let container = null;
-
-            if (file_objs.length > 1) {
-                // create playlist
-                container = await files_api.createPlaylist(download['title'], file_objs.map(file_obj => file_obj.uid), download['user_uid']);
-            } else if (file_objs.length === 1) {
-                container = file_objs[0];
-            } else {
-                const error_message = 'Downloaded file failed to result in metadata object.';
-                logger.error(error_message);
-                await handleDownloadError(download_uid, error_message, 'no_metadata');
-            }
-
-            const file_uids = file_objs.map(file_obj => file_obj.uid);
-            await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
-            resolve(file_uids);
-        }
+        });
     });
 }
 
 // helper functions
 
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
-    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
 
     if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
         logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
@@ -557,30 +550,58 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 }
 
 exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
-    // remove bad args
-    const temp_args = utils.filterArgs(args, ['--no-simulate']);
-    const new_args = [...temp_args];
+    return new Promise(resolve => {
+        // remove bad args
+        const temp_args = utils.filterArgs(args, ['--no-simulate']);
+        const new_args = [...temp_args];
 
-    const archiveArgIndex = new_args.indexOf('--download-archive');
-    if (archiveArgIndex !== -1) {
-        new_args.splice(archiveArgIndex, 2);
-    }
-
-    new_args.push('--dump-json');
-
-    let {callback} = await youtubedl_api.runYoutubeDL(url, new_args);
-    const {parsed_output, err} = await callback;
-    if (!parsed_output || parsed_output.length === 0) {
-        let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
-        if (err.stderr) error_message += `\n\n${err.stderr}`;
-        logger.error(error_message);
-        if (download_uid) {
-            await handleDownloadError(download_uid, error_message, 'info_retrieve_failed');
+        const archiveArgIndex = new_args.indexOf('--download-archive');
+        if (archiveArgIndex !== -1) {
+            new_args.splice(archiveArgIndex, 2);
         }
-        return null;
-    }
 
-    return parsed_output;
+        new_args.push('--dump-json');
+
+        youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
+            if (output) {
+                let outputs = [];
+                try {
+                    for (let i = 0; i < output.length; i++) {
+                        let output_json = null;
+                        try {
+                            output_json = JSON.parse(output[i]);
+                        } catch(e) {
+                            output_json = null;
+                        }
+    
+                        if (!output_json) {
+                            continue;
+                        }
+
+                        outputs.push(output_json);
+                    }
+                    resolve(outputs.length === 1 ? outputs[0] : outputs);
+                } catch(e) {
+                    const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
+                    logger.error(error);
+                    if (download_uid) {
+                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                        await handleDownloadError(download, error, 'parse_failed');
+                    }
+                    resolve(null);
+                }
+            } else {
+                let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
+                if (err.stderr) error_message += `\n\n${err.stderr}`;
+                logger.error(error_message);
+                if (download_uid) {
+                    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                    await handleDownloadError(download, error_message, 'info_retrieve_failed');
+                }
+                resolve(null);
+            }
+        });
+    });
 }
 
 function filterArgs(args, isAudio) {
@@ -599,7 +620,6 @@ async function checkDownloadPercent(download_uid) {
     */
 
     const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (!download) return;
     const files_to_check_for_progress = download['files_to_check_for_progress'];
     const resulting_file_size = download['expected_file_size'];
 

backend/entrypoint.sh

@@ -1,7 +1,17 @@
-#!/bin/bash
+#!/bin/sh
 set -eu
 
+CMD="npm start"
+
+# if the first arg starts with "-" pass it to program
+if [ "${1#-}" != "$1" ]; then
+  set -- "$CMD" "$@"
+fi
+
 # chown current working directory to current user
-echo "[entrypoint] setup permission, this may take a while"
-find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
-exec gosu "$UID:$GID" "$@"
+if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
+  find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  exec gosu "$UID:$GID" "$0" "$@"
+fi
+
+exec "$@"

backend/files.js

@@ -1,350 +0,0 @@
-const fs = require('fs-extra')
-const path = require('path')
-const { v4: uuid } = require('uuid');
-
-const config_api = require('./config');
-const db_api = require('./db');
-const archive_api = require('./archive');
-const utils = require('./utils')
-const logger = require('./logger');
-
-exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
-    if (!file_object) file_object = generateFileObject(file_path, type);
-    if (!file_object) {
-        logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
-        return false;
-    }
-
-    utils.fixVideoMetadataPerms(file_path, type);
-
-    // add thumbnail path
-    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
-
-    // if category exists, only include essential info
-    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
-
-    // modify duration
-    if (cropFileSettings) {
-        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
-    }
-
-    if (user_uid) file_object['user_uid'] = user_uid;
-    if (sub_id) file_object['sub_id'] = sub_id;
-
-    const file_obj = await registerFileDBManual(file_object);
-
-    // remove metadata JSON if needed
-    if (!config_api.getConfigItem('ytdl_include_metadata')) {
-        utils.deleteJSONFile(file_path, type)
-    }
-
-    return file_obj;
-}
-
-async function registerFileDBManual(file_object) {
-    // add additional info
-    file_object['uid'] = uuid();
-    file_object['registered'] = Date.now();
-    const path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
-
-    await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']})
-
-    return file_object;
-}
-
-function generateFileObject(file_path, type) {
-    const jsonobj = utils.getJSON(file_path, type);
-    if (!jsonobj) {
-        return null;
-    } else if (!jsonobj['_filename']) {
-        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
-        return null;
-    }
-    const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
-    // console.
-    const stats = fs.statSync(true_file_path);
-
-    const file_id = utils.removeFileExtension(path.basename(file_path));
-    const title = jsonobj.title;
-    const url = jsonobj.webpage_url;
-    const uploader = jsonobj.uploader;
-    const upload_date = utils.formatDateString(jsonobj.upload_date);
-
-    const size = stats.size;
-
-    const thumbnail = jsonobj.thumbnail;
-    const duration = jsonobj.duration;
-    const isaudio = type === 'audio';
-    const description = jsonobj.description;
-    const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
-    return file_obj;
-}
-
-exports.importUnregisteredFiles = async () => {
-    const imported_files = [];
-    const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
-
-    // run through check list and check each file to see if it's missing from the db
-    for (let i = 0; i < dirs_to_check.length; i++) {
-        const dir_to_check = dirs_to_check[i];
-        // recursively get all files in dir's path
-        const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
-
-        for (let j = 0; j < files.length; j++) {
-            const file = files[j];
-
-            // check if file exists in db, if not add it
-            const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
-            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
-            if (!file_is_registered) {
-                // add additional info
-                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
-                if (file_obj) {
-                    imported_files.push(file_obj['uid']);
-                    logger.verbose(`Added discovered file to the database: ${file.id}`);
-                } else {
-                    logger.error(`Failed to import ${file['path']} automatically.`);
-                }
-            }
-        }
-    }
-    return imported_files;
-}
-
-exports.addMetadataPropertyToDB = async (property_key) => {
-    try {
-        const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
-        const update_obj = {};
-        for (let i = 0; i < dirs_to_check.length; i++) {
-            const dir_to_check = dirs_to_check[i];
-
-            // recursively get all files in dir's path
-            const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
-            for (let j = 0; j < files.length; j++) {
-                const file = files[j];
-                if (file[property_key]) {
-                    update_obj[file.uid] = {[property_key]: file[property_key]};
-                }
-            }
-        }
-
-        return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj);
-    } catch(err) {
-        logger.error(err);
-        return false;
-    }
-}
-
-exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
-    const first_video = await exports.getVideo(uids[0]);
-    const thumbnailToUse = first_video['thumbnailURL'];
-    
-    let new_playlist = {
-        name: playlist_name,
-        uids: uids,
-        id: uuid(),
-        thumbnailURL: thumbnailToUse,
-        registered: Date.now(),
-        randomize_order: false
-    };
-
-    new_playlist.user_uid = user_uid ? user_uid : undefined;
-
-    await db_api.insertRecordIntoTable('playlists', new_playlist);
-    
-    const duration = await exports.calculatePlaylistDuration(new_playlist);
-    await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
-
-    return new_playlist;
-}
-
-exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
-    let playlist = await db_api.getRecord('playlists', {id: playlist_id});
-
-    if (!playlist) {
-        playlist = await db_api.getRecord('categories', {uid: playlist_id});
-        if (playlist) {
-            const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
-            playlist['uids'] = uids;
-            playlist['auto'] = true;
-        }
-    }
-
-    // converts playlists to new UID-based schema
-    if (playlist && playlist['fileNames'] && !playlist['uids']) {
-        playlist['uids'] = [];
-        logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
-        for (let i = 0; i < playlist['fileNames'].length; i++) {
-            const fileName = playlist['fileNames'][i];
-            const uid = await exports.getVideoUIDByID(fileName, user_uid);
-            if (uid) playlist['uids'].push(uid);
-            else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
-        }
-        exports.updatePlaylist(playlist, user_uid);
-    }
-
-    // prevent unauthorized users from accessing the file info
-    if (require_sharing && !playlist['sharingEnabled']) return null;
-
-    return playlist;
-}
-
-exports.updatePlaylist = async (playlist) => {
-    let playlistID = playlist.id;
-
-    const duration = await exports.calculatePlaylistDuration(playlist);
-    playlist.duration = duration;
-
-    return await db_api.updateRecord('playlists', {id: playlistID}, playlist);
-}
-
-exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
-    let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj);
-
-    if (!success) {
-        success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj);
-    }
-
-    if (!success) {
-        logger.error(`Could not find playlist or category with ID ${playlist_id}`);
-    }
-
-    return success;
-}
-
-exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
-    if (!playlist_file_objs) {
-        playlist_file_objs = [];
-        for (let i = 0; i < playlist['uids'].length; i++) {
-            const uid = playlist['uids'][i];
-            const file_obj = await exports.getVideo(uid);
-            if (file_obj) playlist_file_objs.push(file_obj);
-        }
-    }
-
-    return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
-}
-
-exports.deleteFile = async (uid, blacklistMode = false) => {
-    const file_obj = await exports.getVideo(uid);
-    const type = file_obj.isAudio ? 'audio' : 'video';
-    const folderPath = path.dirname(file_obj.path);
-    const name = file_obj.id;
-    const filePathNoExtension = utils.removeFileExtension(file_obj.path);
-
-    var jsonPath = `${file_obj.path}.info.json`;
-    var altJSONPath = `${filePathNoExtension}.info.json`;
-    var thumbnailPath = `${filePathNoExtension}.webp`;
-    var altThumbnailPath = `${filePathNoExtension}.jpg`;
-
-    jsonPath = path.join(__dirname, jsonPath);
-    altJSONPath = path.join(__dirname, altJSONPath);
-
-    let jsonExists = await fs.pathExists(jsonPath);
-    let thumbnailExists = await fs.pathExists(thumbnailPath);
-
-    if (!jsonExists) {
-        if (await fs.pathExists(altJSONPath)) {
-            jsonExists = true;
-            jsonPath = altJSONPath;
-        }
-    }
-
-    if (!thumbnailExists) {
-        if (await fs.pathExists(altThumbnailPath)) {
-            thumbnailExists = true;
-            thumbnailPath = altThumbnailPath;
-        }
-    }
-
-    let fileExists = await fs.pathExists(file_obj.path);
-
-    if (config_api.descriptors[uid]) {
-        try {
-            for (let i = 0; i < config_api.descriptors[uid].length; i++) {
-                config_api.descriptors[uid][i].destroy();
-            }
-        } catch(e) {
-
-        }
-    }
-
-    let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-    if (useYoutubeDLArchive || file_obj.sub_id) {
-        // get id/extractor from JSON
-
-        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-        let retrievedID = null;
-        let retrievedExtractor = null;
-        if (info_json) {
-            retrievedID = info_json['id'];
-            retrievedExtractor = info_json['extractor'];
-        }
-
-        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-        if (!blacklistMode) {
-            await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id)
-        } else {
-            const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
-            if (!exists_in_archive) {
-                await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id);
-            }
-        }
-    }
-
-    if (jsonExists) await fs.unlink(jsonPath);
-    if (thumbnailExists) await fs.unlink(thumbnailPath);
-
-    await db_api.removeRecord('files', {uid: uid});
-
-    if (fileExists) {
-        await fs.unlink(file_obj.path);
-        if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
-            return false;
-        } else {
-            return true;
-        }
-    } else {
-        // TODO: tell user that the file didn't exist
-        return true;
-    }
-}
-
-// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
-exports.getVideoUIDByID = async (file_id, uuid = null) => {
-    const file_obj = await db_api.getRecord('files', {id: file_id});
-    return file_obj ? file_obj['uid'] : null;
-}
-
-exports.getVideo = async (file_uid) => {
-    return await db_api.getRecord('files', {uid: file_uid});
-}
-
-exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
-    const filter_obj = {user_uid: uuid};
-    const regex = true;
-    if (text_search) {
-        if (regex) {
-            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
-        } else {
-            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
-        }
-    }
-
-    if (favorite_filter) {
-        filter_obj['favorite'] = true;
-    }
-
-    if (sub_id) {
-        filter_obj['sub_id'] = sub_id;
-    }
-
-    if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
-    else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
-    
-    const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search)));
-    const file_count = await db_api.getRecords('files', filter_obj, true);
-
-    return {files, file_count};
-}

backend/notifications.js

@@ -4,12 +4,11 @@ const logger = require('./logger');
 const utils = require('./utils');
 const consts = require('./consts');
 
-const { v4: uuid } = require('uuid');
+const { uuid } = require('uuidv4');
 
 const fetch = require('node-fetch');
 const { gotify } = require("gotify");
-const TelegramBotAPI = require('node-telegram-bot-api');
-let telegram_bot = null;
+const TelegramBot = require('node-telegram-bot-api');
 const REST = require('@discordjs/rest').REST;
 const API = require('@discordjs/core').API;
 const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
@@ -57,7 +56,7 @@ exports.sendNotification = async (notification) => {
         sendGotifyNotification(data);
     }
     if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
-        exports.sendTelegramNotification(data);
+        sendTelegramNotification(data);
     }
     if (config_api.getConfigItem('ytdl_webhook_url')) {
         sendGenericNotification(data);
@@ -65,9 +64,6 @@ exports.sendNotification = async (notification) => {
     if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
         sendDiscordNotification(data);
     }
-    if (config_api.getConfigItem('ytdl_slack_webhook_url')) {
-        sendSlackNotification(data);
-    }
 
     await db_api.insertRecordIntoTable('notifications', notification);
     return notification;
@@ -114,8 +110,6 @@ function notificationEnabled(type) {
     return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
 }
 
-// ntfy
-
 function sendNtfyNotification({body, title, type, url, thumbnail}) {
     logger.verbose('Sending notification to ntfy');
     fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
@@ -130,8 +124,6 @@ function sendNtfyNotification({body, title, type, url, thumbnail}) {
     });
 }
 
-// Gotify
-
 async function sendGotifyNotification({body, title, type, url, thumbnail}) {
     logger.verbose('Sending notification to gotify');
     await gotify({
@@ -150,50 +142,15 @@ async function sendGotifyNotification({body, title, type, url, thumbnail}) {
       });
 }
 
-// Telegram
-
-setupTelegramBot();
-config_api.config_updated.subscribe(change => {
-    const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
-    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
-    if (!use_telegram_api || !bot_token) return;
-    if (!change) return;
-    if (change['key'] === 'ytdl_use_telegram_API' || change['key'] === 'ytdl_telegram_bot_token' || change['key'] === 'ytdl_telegram_webhook_proxy') {
-        logger.debug('Telegram bot setting up');
-        setupTelegramBot();
-    }
-});
-
-async function setupTelegramBot() {
-    const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
-    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
-    if (!use_telegram_api || !bot_token) return;
-    
-    telegram_bot = new TelegramBotAPI(bot_token);
-    const webhook_proxy = config_api.getConfigItem('ytdl_telegram_webhook_proxy');
-    const webhook_url = webhook_proxy ? webhook_proxy : `${utils.getBaseURL()}/api/telegramRequest`;
-    telegram_bot.setWebHook(webhook_url);
-}
-
-exports.sendTelegramNotification = async ({body, title, type, url, thumbnail}) => {
-    if (!telegram_bot){
-        logger.error('Telegram bot not found!');
-        return;
-    }
-
-    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
-    if (!chat_id){
-        logger.error('Telegram chat ID required!');
-        return;
-    }
-    
+async function sendTelegramNotification({body, title, type, url, thumbnail}) {
     logger.verbose('Sending notification to Telegram');
-    if (thumbnail) await telegram_bot.sendPhoto(chat_id, thumbnail);
-    telegram_bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    const bot = new TelegramBot(bot_token);
+    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
 }
 
-// Discord
-
 async function sendDiscordNotification({body, title, type, url, thumbnail}) {
     const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
     const url_split = discord_webhook_url.split('webhooks/');
@@ -217,69 +174,6 @@ async function sendDiscordNotification({body, title, type, url, thumbnail}) {
     return result;
 }
 
-// Slack
-
-function sendSlackNotification({body, title, type, url, thumbnail}) {
-    const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
-    logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
-    const data = {
-        blocks: [
-            {
-                type: "section",
-                text: {
-                    type: "mrkdwn",
-                    text: `*${title}*`
-                }
-            },
-            {
-                type: "section",
-                text: {
-                    type: "plain_text",
-                    text: body
-                }
-            }
-        ]
-    }
-
-    // add thumbnail if exists
-    if (thumbnail) {
-        data['blocks'].push({
-            type: "image",
-            image_url: thumbnail,
-            alt_text: "notification_thumbnail"
-        });
-    }
-
-    data['blocks'].push(
-        {
-            type: "section",
-            text: {
-                type: "mrkdwn",
-                text: `<${url}|${url}>`
-            }
-        },
-        {
-            type: "context",
-            elements: [
-                {
-                    type: "mrkdwn",
-                    text: `*ID:* ${type}`
-                }
-            ]
-        }
-    );
-
-    fetch(slack_webhook_url, {
-        method: 'POST',
-        headers: {
-            "Content-Type": "application/json"
-        },
-        body: JSON.stringify(data),
-    });
-}
-
-// Generic
-
 function sendGenericNotification(data) {
     const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
     logger.verbose(`Sending generic notification to ${webhook_url}`);

backend/package.json

@@ -4,7 +4,7 @@
   "description": "backend for YoutubeDL-Material",
   "main": "index.js",
   "scripts": {
-    "test": "mocha test --exit -s 1000",
+    "test": "echo \"Error: no test specified\" && exit 1",
     "start": "pm2-runtime --raw pm2.config.js",
     "debug": "set YTDL_MODE=debug && node app.js"
   },
@@ -17,10 +17,6 @@
   "bugs": {
     "url": ""
   },
-  "engines": {
-    "node": "^16",
-    "npm": "6.14.4"
-  },
   "homepage": "",
   "dependencies": {
     "@discordjs/builders": "^1.6.1",
@@ -30,17 +26,15 @@
     "async-mutex": "^0.4.0",
     "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
-    "command-exists": "^1.2.9",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "execa": "^5.1.1",
     "express": "^4.18.2",
     "express-session": "^1.17.3",
     "feed": "^4.2.2",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
     "gotify": "^1.1.0",
-    "jsonwebtoken": "^9.0.0",
+    "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.21",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
@@ -49,10 +43,10 @@
     "mongodb": "^3.6.9",
     "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
-    "node-id3": "^0.2.6",
+    "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
     "node-telegram-bot-api": "^0.61.0",
-    "passport": "^0.6.0",
+    "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.1",
     "passport-ldapauth": "^3.0.1",
@@ -62,10 +56,10 @@
     "read-last-lines": "^1.7.2",
     "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
-    "tree-kill": "^1.2.2",
     "unzipper": "^0.10.10",
-    "uuid": "^9.0.1",
+    "uuidv4": "^6.2.13",
     "winston": "^3.7.2",
-    "xmlbuilder2": "^3.0.2"
+    "xmlbuilder2": "^3.0.2",
+    "youtube-dl": "^3.0.2"
   }
 }

backend/subscriptions.js

@@ -1,26 +1,25 @@
 const fs = require('fs-extra');
 const path = require('path');
+const youtubedl = require('youtube-dl');
 
-const youtubedl_api = require('./youtube-dl');
 const config_api = require('./config');
 const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
-const CONSTS = require('./consts');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 const db_api = require('./db');
 const downloader_api = require('./downloader');
 
-exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
+async function subscribe(sub, user_uid = null) {
     const result_obj = {
         success: false,
         error: ''
     };
     return new Promise(async resolve => {
         // sub should just have url and name. here we will get isPlaylist and path
-        sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
+        sub.isPlaylist = sub.url.includes('playlist');
         sub.videos = [];
 
         let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -33,13 +32,12 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
         }
 
         sub['user_uid'] = user_uid ? user_uid : undefined;
-        await db_api.insertRecordIntoTable('subscriptions', JSON.parse(JSON.stringify(sub)));
+        await db_api.insertRecordIntoTable('subscriptions', sub);
 
-        let success = skip_get_info ? true : await getSubscriptionInfo(sub);
-        exports.writeSubscriptionMetadata(sub);
+        let success = await getSubscriptionInfo(sub);
 
         if (success) {
-            if (!sub.paused) exports.getVideosForSub(sub.id);
+            getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
         }
@@ -63,41 +61,55 @@ async function getSubscriptionInfo(sub) {
         }
     }
 
-    let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
-    const {parsed_output, err} = await callback;
-    if (err) {
-        logger.error(err.stderr);
-        return false;
-    }
-    logger.verbose('Subscribe: got info for subscription ' + sub.id);
-    for (const output_json of parsed_output) {
-        if (!output_json) {
-            continue;
-        }
-
-        if (!sub.name) {
-            if (sub.isPlaylist) {
-                sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
-            } else {
-                sub.name = output_json.uploader;
+    return new Promise(async resolve => {
+        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
+            if (debugMode) {
+                logger.info('Subscribe: got info for subscription ' + sub.id);
             }
-            // if it's now valid, update
-            if (sub.name) {
-                let sub_name = sub.name;
-                const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
-                if (sub_name_exists) sub_name += ` - ${sub.id}`;
-                await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
+            if (err) {
+                logger.error(err.stderr);
+                resolve(false);
+            } else if (output) {
+                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+                    logger.verbose('Could not get info for ' + sub.id);
+                    resolve(false);
+                }
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+                    if (!output_json) {
+                        continue;
+                    }
+                    if (!sub.name) {
+                        if (sub.isPlaylist) {
+                            sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
+                        } else {
+                            sub.name = output_json.uploader;
+                        }
+                        // if it's now valid, update
+                        if (sub.name) {
+                            let sub_name = sub.name;
+                            const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
+                            if (sub_name_exists) sub_name += ` - ${sub.id}`;
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
+                        }
+                    }
+
+                    // TODO: get even more info
+
+                    resolve(true);
+                }
+                resolve(false);
             }
-        }
-
-        return true;
-    }
-
-    return false;
+        });
+    });
 }
 
-exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => {
-    const sub = await exports.getSubscription(sub_id);
+async function unsubscribe(sub, deleteMode, user_uid = null) {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -120,7 +132,6 @@ exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => {
         }
     }
 
-    await killSubDownloads(sub_id, true);
     await db_api.removeRecord('subscriptions', {id: id});
     await db_api.removeAllRecords('files', {sub_id: id});
 
@@ -137,7 +148,7 @@ exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => {
     await db_api.removeAllRecords('archives', {sub_id: sub.id});
 }
 
-exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
+async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
     if (typeof sub === 'string') {
         // TODO: fix bad workaround where sub is a sub_id
         sub = await db_api.getRecord('subscriptions', {sub_id: sub});
@@ -188,13 +199,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (deleteForever) {
-                // ensure video is in the archives
-                const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
-                if (!exists_in_archive) {
-                    await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
-                }
-            } else {
+            const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useArchive && !deleteForever) {
                 await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
             }
             return true;
@@ -205,76 +211,12 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
     }
 }
 
-let current_sub_index = 0; // To keep track of the current subscription
-exports.watchSubscriptionsInterval = async () => {
-    const subscriptions_check_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
-    let parent_interval = setInterval(() => watchSubscriptions(), subscriptions_check_interval*1000);
-    watchSubscriptions();
-    config_api.config_updated.subscribe(change => {
-        if (!change) return;
-        if (change['key'] === 'ytdl_subscriptions_check_interval' || change['key'] === 'ytdl_multi_user_mode') {
-            current_sub_index = 0; // TODO: start after the last sub check
-            logger.verbose('Resetting sub check schedule due to config change');
-            clearInterval(parent_interval);
-            const new_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
-            parent_interval = setInterval(() => watchSubscriptions(), new_interval*1000);
-            watchSubscriptions();
-        }
-    });
-}
-
-async function watchSubscriptions() {
-    const subscription_ids = await getValidSubscriptionsToCheck();
-    if (subscription_ids.length === 0) {
-        logger.info('Skipping subscription check as no valid subscriptions exist.');
-        return;
-    }
-    checkSubscription(subscription_ids[current_sub_index]);
-    current_sub_index = (current_sub_index + 1) % subscription_ids.length;
-}
-
-async function checkSubscription(sub_id) {
-    let sub = await exports.getSubscription(sub_id);
-
-    // don't check the sub if the last check for the same subscription has not completed
-    if (sub.downloading) {
-        logger.verbose(`Subscription: skipped checking ${sub.name} as it's downloading videos.`);
-        return;
-    }
-
-    if (!sub.name) {
-        logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
-        return;
-    }
-
-    await exports.getVideosForSub(sub.id);
-}
-
-async function getValidSubscriptionsToCheck() {
-    const subscriptions = await exports.getAllSubscriptions();
-
-    if (!subscriptions) return;
-
-    // auto pause deprecated streamingOnly mode
-    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
-    exports.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
-
-    const valid_subscription_ids = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly).map(sub => sub.id);
-    return valid_subscription_ids;
-}
-
-exports.getVideosForSub = async (sub_id) => {
-    const sub = await exports.getSubscription(sub_id);
-    if (!sub || sub['downloading']) {
+async function getVideosForSub(sub, user_uid = null) {
+    const latest_sub_obj = await getSubscription(sub.id);
+    if (!latest_sub_obj || latest_sub_obj['downloading']) {
         return false;
     }
 
-    _getVideosForSub(sub);
-    return true;
-}
-
-async function _getVideosForSub(sub) {
-    const user_uid = sub['user_uid'];
     updateSubscriptionProperty(sub, {downloading: true}, user_uid);
 
     // get basePath
@@ -292,52 +234,84 @@ async function _getVideosForSub(sub) {
     // get videos
     logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
-    let {child_process, callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
-    updateSubscriptionProperty(sub, {child_process: child_process}, user_uid);
-    const {parsed_output, err} = await callback;
-    updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid);
-    if (!parsed_output) {
-        logger.error('Subscription check failed!');
-        if (err) logger.error(err);
-        return null;
-    }
+    return new Promise(async resolve => {
+        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
+            // cleanup
+            updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
-    // remove temporary archive file if it exists
-    const archive_path = path.join(appendedBasePath, 'archive.txt');
-    const archive_exists = await fs.pathExists(archive_path);
-    if (archive_exists) {
-        await fs.unlink(archive_path);
-    }
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
 
-    logger.verbose('Subscription: finished check for ' + sub.name);
-    const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
-    return files_to_download;
+            logger.verbose('Subscription: finished check for ' + sub.name);
+            if (err && !output) {
+                logger.error(err.stderr ? err.stderr : err.message);
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
+                    logger.info('An error was encountered with at least one video, backup method will be used.')
+                    try {
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
+                        resolve(files_to_download);
+                    } catch(e) {
+                        logger.error('Backup method failed. See error below:');
+                        logger.error(e);
+                    }
+                } else {
+                    logger.error('Subscription check failed!');
+                }
+                resolve(false);
+            } else if (output) {
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
+                resolve(files_to_download);
+           }
+        });
+    }, err => {
+        logger.error(err);
+        updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+    });
 }
 
-async function handleOutputJSON(output_jsons, sub, user_uid) {
+async function handleOutputJSON(output, sub, user_uid) {
     if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
         await setFreshUploads(sub, user_uid);
         checkVideosForFreshUploads(sub, user_uid);
     }
 
-    if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) {
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
         logger.verbose('No additional videos to download for ' + sub.name);
         return [];
     }
 
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
     const files_to_download = await getFilesToDownload(sub, output_jsons);
-    const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
 
     for (let j = 0; j < files_to_download.length; j++) {
         const file_to_download = files_to_download[j];
         file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
-        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, [file_to_download]);
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
     }
 
     return files_to_download;
 }
 
-exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
+function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -390,11 +364,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
-    // skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
-    const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
-    const archive_count = archive_text.split('\n').length - 1;
-    if (archive_count > 0) {
-        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
+    // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+        logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
         const archive_path = path.join(appendedBasePath, 'archive.txt');
         await fs.writeFile(archive_path, archive_text);
         downloadConfig.push('--download-archive', archive_path);
@@ -432,7 +406,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push('-r', rate_limit);
     }
 
-    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
     if (default_downloader === 'yt-dlp') {
         downloadConfig.push('--no-clean-info-json');
     }
@@ -454,8 +428,11 @@ async function getFilesToDownload(sub, output_jsons) {
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
                 continue;
             }
-            const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
-            if (exists_in_archive) continue;
+            const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useYoutubeDLArchive) {
+                const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+                if (exists_in_archive) continue;
+            }
 
             files_to_download.push(output_json);
         }
@@ -463,66 +440,36 @@ async function getFilesToDownload(sub, output_jsons) {
     return files_to_download;
 }
 
-exports.cancelCheckSubscription = async (sub_id) => {
-    const sub = await exports.getSubscription(sub_id);
-    if (!sub['downloading'] && !sub['child_process']) {
-        logger.error('Failed to cancel subscription check, verify that it is still running!');
-        return false;
-    }
 
-    // if check is ongoing
-    if (sub['child_process']) {
-        const child_process = sub['child_process'];
-        youtubedl_api.killYoutubeDLProcess(child_process);
-    }
-
-    // cancel activate video downloads
-    await killSubDownloads(sub_id);
-
-    return true;
-}
-
-async function killSubDownloads(sub_id, remove_downloads = false) {
-    const sub_downloads = await db_api.getRecords('download_queue', {sub_id: sub_id});
-    for (const sub_download of sub_downloads) {
-        if (sub_download['running'])
-            await downloader_api.cancelDownload(sub_download['uid']);
-        if (remove_downloads)
-            await db_api.removeRecord('download_queue', {uid: sub_download['uid']});
-    }
-}
-
-exports.getSubscriptions = async (user_uid = null) => {
-    // TODO: fix issue where the downloading property may not match getSubscription()
+async function getSubscriptions(user_uid = null) {
     return await db_api.getRecords('subscriptions', {user_uid: user_uid});
 }
 
-exports.getAllSubscriptions = async () => {
+async function getAllSubscriptions() {
     const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
     return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
 }
 
-exports.getSubscription = async (subID) => {
+async function getSubscription(subID) {
     // stringify and parse because we may override the 'downloading' property
     const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
     // now with the download_queue, we may need to override 'downloading'
-    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
+    const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
     if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
     return sub;
 }
 
-exports.getSubscriptionByName = async (subName, user_uid = null) => {
+async function getSubscriptionByName(subName, user_uid = null) {
     return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
 }
 
-exports.updateSubscription = async (sub) => {
+async function updateSubscription(sub) {
     await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
-    exports.writeSubscriptionMetadata(sub);
     return true;
 }
 
-exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
+async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
         await updateSubscriptionProperty(sub, assignment_obj);
     });
@@ -534,16 +481,6 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-exports.writeSubscriptionMetadata = (sub) => {
-    let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
-                                : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const appendedBasePath = getAppendedBasePath(sub, basePath);
-    const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
-    
-    fs.ensureDirSync(appendedBasePath);
-    fs.writeJSONSync(metadata_path, sub);
-}
-
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     if (!sub_files) return;
@@ -572,22 +509,24 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
     logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
     // simulate a download to verify that a better version exists
-    
-    const info = await downloader_api.getVideoInfoByURL(file_obj['url'], downloadConfig);
-    if (info && info.length === 1) {
-        const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
-        if (info[metric_to_compare] > file_obj[metric_to_compare]) {
-            // download new video as the simulated one is better
-            let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
-            const {parsed_output, err} = await callback;
-            if (err) {
-                logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
-            } else if (parsed_output) {
-                logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${info[metric_to_compare]}`);
-                await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: info[metric_to_compare]});
-            }
-        } 
-    }
+    youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
+        if (err) {
+            // video is not available anymore for whatever reason
+        } else if (output) {
+            const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
+            if (output[metric_to_compare] > file_obj[metric_to_compare]) {
+                // download new video as the simulated one is better
+                youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
+                    if (err) {
+                        logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
+                    } else if (output) {
+                        logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
+                    }
+                });
+            } 
+        }
+    });
     await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
@@ -596,3 +535,17 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
 function getAppendedBasePath(sub, base_path) {
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
+
+module.exports = {
+    getSubscription        : getSubscription,
+    getSubscriptionByName  : getSubscriptionByName,
+    getSubscriptions       : getSubscriptions,
+    getAllSubscriptions    : getAllSubscriptions,
+    updateSubscription     : updateSubscription,
+    subscribe              : subscribe,
+    unsubscribe            : unsubscribe,
+    deleteSubscriptionFile : deleteSubscriptionFile,
+    getVideosForSub        : getVideosForSub,
+    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
+    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
+}

backend/tasks.js

@@ -2,16 +2,9 @@ const db_api = require('./db');
 const notifications_api = require('./notifications');
 const youtubedl_api = require('./youtube-dl');
 const archive_api = require('./archive');
-const files_api = require('./files');
-const subscriptions_api = require('./subscriptions');
-const config_api = require('./config');
-const auth_api = require('./authentication/auth');
-const utils = require('./utils');
-const logger = require('./logger');
-const CONSTS = require('./consts');
 
 const fs = require('fs-extra');
-const path = require('path');
+const logger = require('./logger');
 const scheduler = require('node-schedule');
 
 const TASKS = {
@@ -27,7 +20,7 @@ const TASKS = {
         job: null
     },
     missing_db_records: {
-        run: files_api.importUnregisteredFiles,
+        run: db_api.importUnregisteredFiles,
         title: 'Import missing DB records',
         job: null
     },
@@ -53,11 +46,6 @@ const TASKS = {
         run: archive_api.importArchives,
         title: 'Import legacy archives',
         job: null
-    },
-    rebuild_database: {
-        run: rebuildDB,
-        title: 'Rebuild database',
-        job: null
     }
 }
 
@@ -271,73 +259,9 @@ async function autoDeleteFiles(data) {
         logger.info(`Removing ${data['files_to_remove'].length} old files!`);
         for (let i = 0; i < data['files_to_remove'].length; i++) {
             const file_to_remove = data['files_to_remove'][i];
-            await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
+            await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
         }
     }
 }
 
-async function rebuildDB() {
-    await db_api.backupDB();
-    let subs_to_add = await guessSubscriptions(false);
-    subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
-    const users_to_add = await guessUsers();
-    for (const user_to_add of users_to_add) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        
-        const user_exists = await db_api.getRecord('users', {uid: user_to_add});
-        if (!user_exists) {
-            await auth_api.registerUser(user_to_add, user_to_add, 'password');
-            logger.info(`Regenerated user ${user_to_add}`);
-        }
-        
-        const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
-        const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
-        subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
-    }
-
-    for (const sub_to_add of subs_to_add) {
-        const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
-        // TODO: we shouldn't be creating this here
-        const new_sub = Object.assign({}, sub_to_add, {paused: true});
-        if (!sub_exists) {
-            await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
-            logger.info(`Regenerated subscription ${sub_to_add['name']}`);
-        }
-    }
-
-    logger.info(`Importing unregistered files`);
-    await files_api.importUnregisteredFiles();
-}
-
-const guessUsers = async () => {
-    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-    const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
-    return userPaths.map(userPath => path.basename(userPath));
-}
-
-const guessSubscriptions = async (isPlaylist, basePath = null) => {
-    const guessed_subs = [];
-    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
-
-    const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
-    const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
-
-    const subs = await utils.getDirectoriesInDirectory(subsPath);
-    for (const subPath of subs) {
-        const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
-        if (!fs.existsSync(sub_backup_path)) continue;
-
-        try {
-            const sub_backup = fs.readJSONSync(sub_backup_path)
-            delete sub_backup['_id'];
-            guessed_subs.push(sub_backup);
-        } catch(err) {
-            logger.warn(`Failed to reimport subscription in path ${subPath}`)
-            logger.warn(err);
-        }
-    }
-
-    return guessed_subs;
-}
-
 exports.TASKS = TASKS;

backend/test/tests.js

@@ -3,11 +3,7 @@ const assert = require('assert');
 const low = require('lowdb')
 const winston = require('winston');
 const path = require('path');
-const util = require('util');
-const fs = require('fs-extra');
-const { v4: uuid } = require('uuid');
-const NodeID3 = require('node-id3');
-const exec = util.promisify(require('child_process').exec);
+
 
 const FileSync = require('lowdb/adapters/FileSync');
 
@@ -44,12 +40,11 @@ const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
 const archive_api = require('../archive');
 const categories_api = require('../categories');
-const files_api = require('../files');
-const youtubedl_api = require('../youtube-dl');
-const config_api = require('../config');
-const CONSTS = require('../consts');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
 
-db_api.initialize(db, users_db, 'local_db_test.json');
+db_api.initialize(db, users_db);
 
 const sample_video_json = {
     id: "Sample Video",
@@ -72,9 +67,9 @@ const sample_video_json = {
 }
 
 describe('Database', async function() {
-    describe.skip('Import', async function() {
+    describe('Import', async function() {
         // it('Migrate', async function() {
-        //     // await db_api.connectToDB();
+        //     await db_api.connectToDB();
         //     await db_api.removeAllRecords();
         //     const success = await db_api.importJSONToDB(db.value(), users_db.value());
         //     assert(success);
@@ -90,7 +85,7 @@ describe('Database', async function() {
         });
 
         it('Transfer to local', async function() {
-            // await db_api.connectToDB();
+            await db_api.connectToDB();
             await db_api.removeAllRecords('test');
             await db_api.insertRecordIntoTable('test', {test: 'test'});
 
@@ -118,8 +113,7 @@ describe('Database', async function() {
 
         for (const local_db_mode of local_db_modes) {
             let use_local_db = local_db_mode;
-            const describe_skippable = use_local_db ? describe : describe.skip;
-            describe_skippable(`Use local DB - ${use_local_db}`, async function() {
+            describe(`Use local DB - ${use_local_db}`, async function() {
                 beforeEach(async function() {
                     if (!use_local_db) {
                         this.timeout(120000);
@@ -172,7 +166,7 @@ describe('Database', async function() {
                     ];
                     await db_api.insertRecordsIntoTable('test', test_duplicates);
                     const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-                    assert(duplicates && duplicates.length === 2 && duplicates[0]['key'] === '2' && duplicates[1]['key'] === '4')
+                    console.log(duplicates);
                 });
 
                 it('Update record', async function() {
@@ -284,7 +278,7 @@ describe('Database', async function() {
                     assert(stats);
                 });
 
-                it.skip('Query speed', async function() {
+                it('Query speed', async function() {
                     this.timeout(120000); 
                     const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
                     const test_records = [];
@@ -342,13 +336,12 @@ describe('Database', async function() {
 });
 
 describe('Multi User', async function() {
-    this.timeout(120000);
     const user_to_test = 'test_user';
     const user_password = 'test_pass';
     const sub_to_test = '';
     const playlist_to_test = '';
     beforeEach(async function() {
-        // await db_api.connectToDB();
+        await db_api.connectToDB();
         await auth_api.deleteUser(user_to_test);
     });
     describe('Basic', function() {
@@ -369,30 +362,30 @@ describe('Multi User', async function() {
         });
         const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
-            const video_obj = await files_api.getVideo(video_to_test);
+            const video_obj = await db_api.getVideo(video_to_test);
             assert(video_obj);
         });
 
         it('Video access - disallowed', async function() {
             await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
-            const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(!video_obj);
         });
 
         it('Video access - allowed', async function() {
             await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
-            const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
+            const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
             assert(video_obj);
         });
     });
-    describe.skip('Zip generators', function() {
+    describe('Zip generators', function() {
         it('Playlist zip generator', async function() {
-            const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
+            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
             assert(playlist);
             const playlist_files_to_download = [];
             for (let i = 0; i < playlist['uids'].length; i++) {
                 const uid = playlist['uids'][i];
-                const playlist_file = await files_api.getVideo(uid, user_to_test);
+                const playlist_file = await db_api.getVideo(uid, user_to_test);
                 playlist_files_to_download.push(playlist_file);
             }
             const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
@@ -402,7 +395,7 @@ describe('Multi User', async function() {
         });
 
         it('Subscription zip generator', async function() {
-            const sub = await subscriptions_api.getSubscription(sub_to_test.id, user_to_test);
+            const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
             const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
             assert(sub);
             const sub_files_to_download = [];
@@ -420,7 +413,7 @@ describe('Multi User', async function() {
     //     const sub_to_test = '';
     //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
     //     it('Get video', async function() {
-    //         const video_obj = files_api.getVideo(video_to_test, 'admin', );
+    //         const video_obj = db_api.getVideo(video_to_test, 'admin', );
     //         assert(video_obj);
     //     });
 
@@ -441,100 +434,35 @@ describe('Multi User', async function() {
     
 describe('Downloader', function() {
     const downloader_api = require('../downloader');
-    const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
-    const playlist_url = 'https://www.youtube.com/playlist?list=PLbZT16X07RLhqK-ZgSkRuUyiz9B_WLdNK';
+    const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
     const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
     const options = {
-        ui_uid: uuid()
+        ui_uid: uuid(),
+        user: 'admin'
     }
 
-    async function createCategory(url) {
-        // get info
-        const args = await downloader_api.generateArgs(url, 'video', options, null, true);
-        const [info] = await downloader_api.getVideoInfoByURL(url, args);
-
-        // create category
-        await db_api.removeAllRecords('categories');
-        const new_category = {
-            name: 'test_category',
-            uid: uuid(),
-            rules: [],
-            custom_output: ''
-        };
-        await db_api.insertRecordIntoTable('categories', new_category);
-        await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
-            preceding_operator: null,
-            comparator: 'includes',
-            property: 'title',
-            value: info['title']
-        });
-    }
-
-    before(async function() {
-        const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
-        if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
-        config_api.setConfigItem('ytdl_max_concurrent_downloads', 0);
-    });
-
     beforeEach(async function() {
-        // await db_api.connectToDB();
+        await db_api.connectToDB();
         await db_api.removeAllRecords('download_queue');
-        config_api.setConfigItem('ytdl_allow_playlist_categorization', true);
     });
 
     it('Get file info', async function() {
         this.timeout(300000);
         const info = await downloader_api.getVideoInfoByURL(url);
-        assert(!!info && info.length > 0);
+        assert(!!info);
     });
 
     it('Download file', async function() {
-        this.timeout(300000);
-        await downloader_api.setupDownloads();
-        const args = await downloader_api.generateArgs(url, 'video', options, null, true);
-        const [info] = await downloader_api.getVideoInfoByURL(url, args);
-        if (fs.existsSync(info['_filename'])) fs.unlinkSync(info['_filename']);
+        this.timeout(300000); 
         const returned_download = await downloader_api.createDownload(url, 'video', options);
-        assert(returned_download);
-        const custom_download_method = async (url, args, options, callback) => {
-            fs.writeJSONSync(utils.getTrueFileName(info['_filename'], 'video', '.info.json'), info);
-            await generateEmptyVideoFile(info['_filename']);
-            return await callback(null, [JSON.stringify(info)]);
-        }
-        const success = await downloader_api.downloadQueuedFile(returned_download['uid'], custom_download_method);
-        assert(success);
-    });
+        console.log(returned_download);
+        await utils.wait(20000);
 
-    it('Downloader - categorize', async function() {
-        this.timeout(300000);
-        await createCategory(url);
-        // collect info
-        const returned_download = await downloader_api.createDownload(url, 'video', options);
-        await downloader_api.collectInfo(returned_download['uid']);
-        assert(returned_download['category']);
-        assert(returned_download['category']['name'] === 'test_category');
-    });
-
-    it('Downloader - categorize playlist', async function() {
-        this.timeout(300000);
-        await createCategory(playlist_url);
-        // collect info
-        const returned_download_pass = await downloader_api.createDownload(playlist_url, 'video', options);
-        await downloader_api.collectInfo(returned_download_pass['uid']);
-        assert(returned_download_pass['category']);
-        assert(returned_download_pass['category']['name'] === 'test_category');
-
-        // test with playlist categorization disabled
-        config_api.setConfigItem('ytdl_allow_playlist_categorization', false);
-        const returned_download_fail = await downloader_api.createDownload(playlist_url, 'video', options);
-        await downloader_api.collectInfo(returned_download_fail['uid']);
-        assert(!returned_download_fail['category']);
     });
 
     it('Tag file', async function() {
-        const success = await generateEmptyAudioFile('test/sample_mp3.mp3');
-        const audio_path = './test/sample_mp3.mp3';
-        const sample_json = fs.readJSONSync('./test/sample_mp3.info.json');
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
         const tags = {
             title: sample_json['title'],
             artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
@@ -542,13 +470,14 @@ describe('Downloader', function() {
         }
         NodeID3.write(tags, audio_path);
         const written_tags = NodeID3.read(audio_path);
-        assert(success && written_tags['raw']['TRCK'] === '27');
+        assert(written_tags['raw']['TRCK'] === '27');
     });
 
     it('Queue file', async function() {
         this.timeout(300000); 
-        const returned_download = await downloader_api.createDownload(url, 'video', options, null, null, null, null, true);
-        assert(returned_download);
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        console.log(returned_download);
+        await utils.wait(20000);
     });
 
     it('Pause file', async function() {
@@ -563,7 +492,7 @@ describe('Downloader', function() {
         assert(args.length > 0);
     });
 
-    it.skip('Generate args - subscription', async function() {
+    it('Generate args - subscription', async function() {
         const sub = await subscriptions_api.getSubscription(sub_id);
         const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
         const args_normal = await downloader_api.generateArgs(url, 'video', options);
@@ -576,7 +505,7 @@ describe('Downloader', function() {
         if (fs.existsSync(nfo_file_path)) {
             fs.unlinkSync(nfo_file_path);
         }
-        const sample_json = fs.readJSONSync('./test/sample_mp4.info.json');
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
         downloader_api.generateNFOFile(sample_json, nfo_file_path);
         assert(fs.existsSync(nfo_file_path), true);
         fs.unlinkSync(nfo_file_path);
@@ -603,19 +532,11 @@ describe('Downloader', function() {
     });
     describe('Twitch', async function () {
         const twitch_api = require('../twitch');
-        const example_vod = '1790315420';
-        it('Download VOD chat', async function() {
-            this.timeout(300000);
-            if (!fs.existsSync('TwitchDownloaderCLI')) {
-                try {
-                    await exec('sh ../docker-utils/fetch-twitchdownloader.sh');
-                    fs.copyFileSync('../docker-utils/TwitchDownloaderCLI', 'TwitchDownloaderCLI');
-                } catch (e) {
-                    logger.info('TwitchDownloaderCLI fetch failed, file may exist regardless.');
-                }
-            }
+        const example_vod = '1710641401';
+        it('Download VOD', async function() {
             const sample_path = path.join('test', 'sample.twitch_chat.json');
             if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
             await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
             assert(fs.existsSync(sample_path));
 
@@ -625,109 +546,10 @@ describe('Downloader', function() {
     });
 });
 
-describe('youtube-dl', async function() {
-    beforeEach(async function () {
-        if (fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.unlinkSync(CONSTS.DETAILS_BIN_PATH);
-        await youtubedl_api.checkForYoutubeDLUpdate();
-    });
-    it('Check latest version', async function() {
-        this.timeout(300000);
-        const original_fork = config_api.getConfigItem('ytdl_default_downloader');
-        const latest_version = await youtubedl_api.getLatestUpdateVersion(original_fork);
-        assert(latest_version > CONSTS.OUTDATED_YOUTUBEDL_VERSION);
-    });
-
-    it('Update youtube-dl', async function() {
-        this.timeout(300000);
-        const original_fork = config_api.getConfigItem('ytdl_default_downloader');
-        const binary_path = path.join('test', 'test_binary');
-        for (const youtubedl_fork in youtubedl_api.youtubedl_forks) {
-            config_api.setConfigItem('ytdl_default_downloader', youtubedl_fork);
-            const latest_version = await youtubedl_api.checkForYoutubeDLUpdate();
-            await youtubedl_api.updateYoutubeDL(latest_version, binary_path);
-            assert(fs.existsSync(binary_path));
-            if (fs.existsSync(binary_path)) fs.unlinkSync(binary_path);
-        }
-        config_api.setConfigItem('ytdl_default_downloader', original_fork);
-    });
-
-    it('Run process', async function() {
-        this.timeout(300000);
-        const downloader_api = require('../downloader');
-        const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
-        const args = await downloader_api.generateArgs(url, 'video', {}, null, true);
-        const {child_process} = await youtubedl_api.runYoutubeDL(url, args);
-        assert(child_process);
-    });
-});
-
-describe('Subscriptions', function() {
-    const new_sub = {
-        name: 'test_sub',
-        url: 'https://www.youtube.com/channel/UCzofo-P8yMMCOv8rsPfIR-g',
-        maxQuality: null,
-        id: uuid(),
-        user_uid: null,
-        type: 'video',
-        paused: true
-    };
-    beforeEach(async function() {
-        await db_api.removeAllRecords('subscriptions');
-    });
-    it('Subscribe', async function () {
-        const success = await subscriptions_api.subscribe(new_sub, null, true);
-        assert(success);
-        const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
-        assert(sub_exists);
-    });
-    it('Unsubscribe', async function () {
-        await subscriptions_api.subscribe(new_sub, null, true);
-        await subscriptions_api.unsubscribe(new_sub);
-        const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']});
-        assert(!sub_exists);
-    });
-    it('Delete subscription file', async function () {
-        
-    });
-    it('Get subscription by name', async function () {
-        await subscriptions_api.subscribe(new_sub, null, true);
-        const sub_by_name = await subscriptions_api.getSubscriptionByName('test_sub');
-        assert(sub_by_name);
-    });
-    it('Get subscriptions', async function() {
-        await subscriptions_api.subscribe(new_sub, null, true);
-        const subs = await subscriptions_api.getSubscriptions(null);
-        assert(subs && subs.length === 1);
-    });
-    it('Update subscription', async function () {
-        await subscriptions_api.subscribe(new_sub, null, true);
-        const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
-        await subscriptions_api.updateSubscription(sub_update);
-        const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
-        assert(updated_sub['name'] === 'updated_name');
-    });
-    it('Update subscription property', async function () {
-        await subscriptions_api.subscribe(new_sub, null, true);
-        const sub_update = Object.assign({}, new_sub, {name: 'updated_name'});
-        await subscriptions_api.updateSubscriptionPropertyMultiple([sub_update], {name: 'updated_name'});
-        const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']});
-        assert(updated_sub['name'] === 'updated_name');
-    });
-    it('Write subscription metadata', async function() {
-        const metadata_path = path.join('subscriptions', 'channels', 'test_sub', 'subscription_backup.json');
-        if (fs.existsSync(metadata_path)) fs.unlinkSync(metadata_path);
-        await subscriptions_api.subscribe(new_sub, null, true);
-        assert(fs.existsSync(metadata_path));
-    });
-    it('Fresh uploads', async function() {
-
-    });
-});
-
 describe('Tasks', function() {
     const tasks_api = require('../tasks');
     beforeEach(async function() {
-        // await db_api.connectToDB();
+        await db_api.connectToDB();
         await db_api.removeAllRecords('tasks');
 
         const dummy_task = {
@@ -746,7 +568,7 @@ describe('Tasks', function() {
         await tasks_api.executeTask('backup_local_db');
         const backups_new = await utils.recFindByExt('appdata', 'bak');
         const new_length = backups_new.length;
-        assert(original_length === new_length-1);
+        assert(original_length, new_length-1);
     });
 
     it('Check for missing files', async function() {
@@ -756,7 +578,7 @@ describe('Tasks', function() {
         await db_api.insertRecordIntoTable('files', test_missing_file);
         await tasks_api.executeTask('missing_files_check');
         const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
-        assert(!missing_file_db_record);
+        assert(!missing_file_db_record, true);
     });
 
     it('Check for duplicate files', async function() {
@@ -776,29 +598,27 @@ describe('Tasks', function() {
 
         await tasks_api.executeTask('duplicate_files_check');
         const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
-        assert(duplicated_record_count === 1);
+        assert(duplicated_record_count == 1, true);
     });
 
     it('Import unregistered files', async function() {
         this.timeout(300000);
 
-        const success = await generateEmptyVideoFile('test/sample_mp4.mp4');
-
         // pre-test cleanup
-        await db_api.removeAllRecords('files', {path: 'test/missing_file.mp4'});
-        if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
-        if (fs.existsSync('video/sample_mp4.mp4'))       fs.unlinkSync('video/sample_mp4.mp4');
+        await db_api.removeAllRecords('files', {title: 'Sample File'});
+        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
+        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
 
         // copies in files
-        fs.copyFileSync('test/sample_mp4.info.json', 'video/sample_mp4.info.json');
-        fs.copyFileSync('test/sample_mp4.mp4', 'video/sample_mp4.mp4');
+        fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
+        fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
         await tasks_api.executeTask('missing_db_records');
         const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(success && !!imported_file);
+        assert(!!imported_file === true);
         
         // post-test cleanup
-        if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
-        if (fs.existsSync('video/sample_mp4.mp4'))       fs.unlinkSync('video/sample_mp4.mp4');
+        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
+        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
     });
 
     it('Schedule and cancel task', async function() {
@@ -838,12 +658,12 @@ describe('Tasks', function() {
 
 describe('Archive', async function() {
     beforeEach(async function() {
-        // await db_api.connectToDB();
-        await db_api.removeAllRecords('archives');
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
     });
 
     afterEach(async function() {
-        await db_api.removeAllRecords('archives');
+        await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
     });
 
     it('Import archive', async function() {
@@ -857,6 +677,7 @@ describe('Archive', async function() {
         const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
         assert(count === 4)
         const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
+        console.log(archive_items);
         assert(archive_items.length === 4);
         assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
         assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
@@ -887,9 +708,9 @@ describe('Archive', async function() {
     });
 
     it('Remove from archive', async function() {
-        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_title', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_title', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_title', 'test_user');
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
 
         const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
         assert(success);
@@ -935,14 +756,14 @@ describe('Utils', async function() {
 
 describe('Categories', async function() {
     beforeEach(async function() {
-        // await db_api.connectToDB();
+        await db_api.connectToDB();
         const new_category = {
             name: 'test_category',
             uid: uuid(),
             rules: [],
             custom_output: ''
         };
-        await db_api.removeAllRecords('categories', {name: 'test_category'});
+    
         await db_api.insertRecordIntoTable('categories', new_category);
     });
 
@@ -983,6 +804,7 @@ describe('Categories', async function() {
         });
 
         const category = await categories_api.categorize([sample_video_json]);
+        console.log(category);
         assert(category && category.name === 'test_category');
     });
 
@@ -1035,74 +857,4 @@ describe('Categories', async function() {
         const category = await categories_api.categorize([sample_video_json]);
         assert(category);
     });
-});
-
-describe('Config', async function() {
-    it('findChangedConfigItems', async function() {
-        const old_config = {
-            "YoutubeDLMaterial": {
-                "test_object1": {
-                    "test_prop1": true,
-                    "test_prop2": false
-                },
-                "test_object2": {
-                    "test_prop3": {
-                        "test_prop3_1": true,
-                        "test_prop3_2": false
-                    },
-                    "test_prop4": false
-                },
-                "test_object3": {
-                    "test_prop5": {
-                        "test_prop5_1": true,
-                        "test_prop5_2": false
-                    },
-                    "test_prop6": false
-                }
-            }
-        };
-
-        const new_config = {
-            "YoutubeDLMaterial": {
-                "test_object1": {
-                    "test_prop1": false,
-                    "test_prop2": false
-                },
-                "test_object2": {
-                    "test_prop3": {
-                        "test_prop3_1": false,
-                        "test_prop3_2": false
-                    },
-                    "test_prop4": true
-                },
-                "test_object3": {
-                    "test_prop5": {
-                        "test_prop5_1": true,
-                        "test_prop5_2": false
-                    },
-                    "test_prop6": true
-                }
-            }
-        };
-
-        const changes = config_api.findChangedConfigItems(old_config, new_config);
-        assert(changes[0]['key'] === 'test_prop1' && changes[0]['old_value'] === true && changes[0]['new_value'] === false);
-        assert(changes[1]['key'] === 'test_prop3' &&
-                changes[1]['old_value']['test_prop3_1'] === true &&
-                changes[1]['new_value']['test_prop3_1'] === false &&
-                changes[1]['old_value']['test_prop3_2'] === false &&
-                changes[1]['new_value']['test_prop3_2'] === false);
-        assert(changes[2]['key'] === 'test_prop4' && changes[2]['old_value'] === false && changes[2]['new_value'] === true);
-        assert(changes[3]['key'] === 'test_prop6' && changes[3]['old_value'] === false && changes[3]['new_value'] === true);
-    });
-});
-
-const generateEmptyVideoFile = async (file_path) => {
-    if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
-    return await exec(`ffmpeg -t 1 -f lavfi -i color=c=black:s=640x480 -c:v libx264 -tune stillimage -pix_fmt yuv420p "${file_path}"`);
-}
-
-const generateEmptyAudioFile = async (file_path) => {
-    if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
-    return await exec(`ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 1 -q:a 9 -acodec libmp3lame ${file_path}`);
-}
+});

backend/twitch.js

@@ -6,7 +6,6 @@ const fs = require('fs-extra')
 const path = require('path');
 const { promisify } = require('util');
 const child_process = require('child_process');
-const commandExistsSync = require('command-exists').sync;
 
 async function getCommentsForVOD(vodId) {
     const exec = promisify(child_process.exec);
@@ -21,12 +20,12 @@ async function getCommentsForVOD(vodId) {
     const cliExt = is_windows ? '.exe' : ''
     const cliPath = `TwitchDownloaderCLI${cliExt}`
 
-    if (!commandExistsSync(cliPath)) {
+    if (!fs.existsSync(cliPath)) {
         logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
         return null;
     }
 
-    const result = await exec(`${cliPath} chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
+    const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
 
     if (result['stderr']) {
         logger.error(`Failed to download twitch comments for ${vodId}`);

backend/utils.js

@@ -13,7 +13,7 @@ const CONSTS = require('./consts');
 const is_windows = process.platform === 'win32';
 
 // replaces .webm with appropriate extension
-exports.getTrueFileName = (unfixed_path, type, force_ext = null) => {
+exports.getTrueFileName = (unfixed_path, type) => {
     let fixed_path = unfixed_path;
 
     const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -22,7 +22,7 @@ exports.getTrueFileName = (unfixed_path, type, force_ext = null) => {
 
 
     if (old_ext !== new_ext) {
-        unfixed_parts[unfixed_parts.length-1] = force_ext || new_ext;
+        unfixed_parts[unfixed_parts.length-1] = new_ext;
         fixed_path = unfixed_parts.join('.');
     }
     return fixed_path;
@@ -241,6 +241,11 @@ exports.addUIDsToCategory = (category, files) => {
     return files_that_match;
 }
 
+exports.getCurrentDownloader = () => {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    return details_json['downloader'];
+}
+
 exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
     files = files || (await fs.readdir(base))
     result = result || []
@@ -342,7 +347,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
             if (!err) {
                 clearTimeout(timer);
                 if (watcher) watcher.close();
-                resolve(true);
+                resolve();
             }
         });
 
@@ -352,7 +357,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
             if (eventType === 'rename' && filename === basename) {
                 clearTimeout(timer);
                 if (watcher) watcher.close();
-                resolve(true);
+                resolve();
             }
         });
     });
@@ -514,51 +519,6 @@ exports.convertFlatObjectToNestedObject = (obj) => {
     return result;
 }
 
-exports.getDirectoriesInDirectory = async (basePath) => {
-    try {
-        const files = await fs.readdir(basePath, { withFileTypes: true });
-        return files
-            .filter((file) => file.isDirectory())
-            .map((file) => path.join(basePath, file.name));
-    } catch (err) {
-        return [];
-    }
-}
-
-exports.parseOutputJSON = (output, err) => {
-    let split_output = [];
-    // const output_jsons = [];
-    if (err && !output) {
-        if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) {
-            return null;
-        }
-        logger.info('An error was encountered with at least one video, backup method will be used.')
-        try {
-            split_output = err.stdout.split(/\r\n|\r|\n/);
-        } catch (e) {
-            logger.error('Backup method failed. See error below:');
-            logger.error(e);
-            return null;
-        }
-    } else if (output.length === 0 || (output.length === 1 && output[0].length === 0)) {
-        // output is '' or ['']
-        return [];
-    } else {
-        for (const output_item of output) {
-            // we have to do this because sometimes there will be leading characters before the actual json
-            const start_idx = output_item.indexOf('{"');
-            const clean_output = output_item.slice(start_idx, output_item.length);
-            split_output.push(clean_output);
-        }
-    }
-
-    try {
-        return split_output.map(split_output_str => JSON.parse(split_output_str));
-    } catch(e) {
-        return null;
-    }
-}
-
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {

backend/youtube-dl.js

@@ -1,159 +1,141 @@
 const fs = require('fs-extra');
 const fetch = require('node-fetch');
-const path = require('path');
-const execa = require('execa');
-const kill = require('tree-kill');
 
 const logger = require('./logger');
 const utils = require('./utils');
 const CONSTS = require('./consts');
 const config_api = require('./config.js');
 
+const OUTDATED_VERSION = "2020.00.00";
+
 const is_windows = process.platform === 'win32';
 
-exports.youtubedl_forks = {
+const download_sources = {
     'youtube-dl': {
-        'download_url': 'https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl',
-        'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'
+        'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
+        'func': downloadLatestYoutubeDLBinary
     },
     'youtube-dlc': {
-        'download_url': 'https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc',
-        'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags'
+        'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
+        'func': downloadLatestYoutubeDLCBinary
     },
     'yt-dlp': {
-        'download_url': 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp',
-        'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags'
+        'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
+        'func': downloadLatestYoutubeDLPBinary
     }
 }
 
-exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => {
-    const output_file_path = getYoutubeDLPath();
-    if (!fs.existsSync(output_file_path)) await exports.checkForYoutubeDLUpdate();
-    let callback = null;
-    let child_process = null;
-    if (customDownloadHandler) {
-        callback = runYoutubeDLCustom(url, args, customDownloadHandler);
-    } else {
-        ({callback, child_process} = await runYoutubeDLProcess(url, args));
-    }
-
-    return {child_process, callback};
-}
-
-// Run youtube-dl directly (not cancellable)
-const runYoutubeDLCustom = async (url, args, customDownloadHandler) => {
-    const downloadHandler = customDownloadHandler;
-    return new Promise(resolve => {
-        downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) {
-            const parsed_output = utils.parseOutputJSON(output, err);
-            resolve({parsed_output, err});
-        });
-    });
-}
-
-// Run youtube-dl in a subprocess (cancellable)
-const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => {
-    const youtubedl_path = getYoutubeDLPath(youtubedl_fork);
-    const binary_exists = fs.existsSync(youtubedl_path);
-    if (!binary_exists) {
-        const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`;
-        logger.error(err);
-        return;
-    }
-    const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity});
-    const callback = new Promise(async resolve => {
-        try {
-            const {stdout, stderr} = await child_process;
-            const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr);
-            resolve({parsed_output, err: stderr});
-        } catch (e) {
-            resolve({parsed_output: null, err: e})
-        }
-    });
-    return {child_process, callback}
-}
-
-function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) {
-    const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : '');
-    const binary_path = path.join('appdata', 'bin', binary_file_name);
-    return binary_path;
-}
-
-exports.killYoutubeDLProcess = async (child_process) => {
-    kill(child_process.pid, 'SIGKILL');
-}
-
 exports.checkForYoutubeDLUpdate = async () => {
-    const selected_fork = config_api.getConfigItem('ytdl_default_downloader');
-    const output_file_path = getYoutubeDLPath();
-    // get current version
-    let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
-    if (!current_app_details_exists[selected_fork]) {
-        logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
-        updateDetailsJSON(CONSTS.OUTDATED_YOUTUBEDL_VERSION, selected_fork, output_file_path);
-    }
-    const current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
-    const current_version = current_app_details[selected_fork]['version'];
-    const current_fork = current_app_details[selected_fork]['downloader'];
+    return new Promise(async resolve => {
+        const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+        const tags_url = download_sources[default_downloader]['tags_url'];
+        // get current version
+        let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
+        if (!current_app_details_exists) {
+            logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
+            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader});
+        }
+        let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
+        let current_version = current_app_details['version'];
+        let current_downloader = current_app_details['downloader'];
+        let stored_binary_path = current_app_details['path'];
+        if (!stored_binary_path || typeof stored_binary_path !== 'string') {
+            // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
+            const guessed_base_path = 'node_modules/youtube-dl/bin/';
+            const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
+            if (fs.existsSync(guessed_file_path)) {
+                stored_binary_path = guessed_file_path;
+                // logger.info('INFO: Guess successful! Update process continuing...')
+            } else {
+                logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
+                resolve(null);
+                return;
+            }
+        }
 
-    const latest_version = await exports.getLatestUpdateVersion(selected_fork);
-    // if the binary does not exist, or default_downloader doesn't match existing fork, or if the fork has been updated, redownload
-    // TODO: don't redownload if fork already exists
-    if (!fs.existsSync(output_file_path) || current_fork !== selected_fork || !current_version || current_version !== latest_version) {
-        logger.warn(`Updating ${selected_fork} binary to '${output_file_path}', downloading...`);
-        await exports.updateYoutubeDL(latest_version);
-    }
-}
+        // got version, now let's check the latest version from the youtube-dl API
 
-exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => {
-    await fs.ensureDir(path.join('appdata', 'bin'));
-    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
-    await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path);
-}
 
-async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) {
-    const file_ext = is_windows ? '.exe' : '';
-
-    // build the URL
-    const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
-    const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork);
-
-    await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`);
-    fs.chmod(output_path, 0o777);
-
-    updateDetailsJSON(new_version, youtubedl_fork, output_path);
-} 
-
-exports.getLatestUpdateVersion = async (youtubedl_fork) => {
-    const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url'];
-    return new Promise(resolve => {
         fetch(tags_url, {method: 'Get'})
         .then(async res => res.json())
         .then(async (json) => {
+            // check if the versions are different
             if (!json || !json[0]) {
-                logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
+                logger.error(`Failed to check ${default_downloader} version for an update.`)
                 resolve(null);
                 return;
             }
             const latest_update_version = json[0]['name'];
-            resolve(latest_update_version);
+            if (current_version !== latest_update_version || default_downloader !== current_downloader) {
+                // versions different or different downloader is being used, download new update
+                resolve(latest_update_version);
+            } else {
+                resolve(null);
+            }
+            return;
         })
         .catch(err => {
-            logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
+            logger.error(`Failed to check ${default_downloader} version for an update.`)
             logger.error(err);
             resolve(null);
+            return;
         });
     });
 }
 
-function updateDetailsJSON(new_version, fork, output_path) {
+exports.updateYoutubeDL = async (latest_update_version) => {
+    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+    await download_sources[default_downloader]['func'](latest_update_version);
+}
+
+exports.verifyBinaryExistsLinux = () => {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
+        details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
+        details_json['exec'] = 'youtube-dl';
+        details_json['version'] = OUTDATED_VERSION;
+        fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
+
+        utils.restartServer();
+    }
+}
+
+async function downloadLatestYoutubeDLBinary(new_version) {
     const file_ext = is_windows ? '.exe' : '';
-    const details_json = fs.existsSync(CONSTS.DETAILS_BIN_PATH) ? fs.readJSONSync(CONSTS.DETAILS_BIN_PATH) : {};
-    if (!details_json[fork]) details_json[fork] = {};
-    const fork_json = details_json[fork];
-    fork_json['version'] = new_version;
-    fork_json['downloader'] = fork;
-    fork_json['path'] = output_path; // unused
-    fork_json['exec'] = fork + file_ext; // unused
+
+    const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
+
+    updateDetailsJSON(new_version, 'youtube-dl');
+}
+
+async function downloadLatestYoutubeDLCBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
+
+    updateDetailsJSON(new_version, 'youtube-dlc');
+}
+
+async function downloadLatestYoutubeDLPBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
+
+    updateDetailsJSON(new_version, 'yt-dlp');
+}
+
+function updateDetailsJSON(new_version, downloader) {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (new_version) details_json['version'] = new_version;
+    details_json['downloader'] = downloader;
     fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
 }

chart/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: youtubedl-material
-description: A Helm chart for https://github.com/Tzahi12345/YoutubeDL-Material
+description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.3.2"
+appVersion: "4.3.1"

chart/templates/ingress.yaml

@@ -1,14 +1,7 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "youtubedl-material.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
-  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
-  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
-  {{- end }}
-{{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1beta1
 {{- else -}}
 apiVersion: extensions/v1beta1
@@ -23,9 +16,6 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.className }}
-  {{- end }}
   {{- if .Values.ingress.tls }}
   tls:
     {{- range .Values.ingress.tls }}
@@ -43,19 +33,9 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-            pathType: {{ .pathType }}
-            {{- end }}
             backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}
-                port:
-                  number: {{ $svcPort }}
-              {{- else }}
               serviceName: {{ $fullName }}
               servicePort: {{ $svcPort }}
-              {{- end }}
           {{- end }}
     {{- end }}
-{{- end }}
+  {{- end }}

docker-compose.yml

@@ -18,7 +18,6 @@ services:
             - "8998:17442"
         image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
-        # If you are using a Raspberry Pi, use mongo:4.4.18
         image: mongo:4
         logging:
             driver: "none"          

docker-utils/GetTwitchDownloader.py

@@ -3,26 +3,13 @@ import requests
 import shutil
 import os
 import re
-import sys
-from collections import OrderedDict
 
 from github import Github
 
 machine = platform.machine()
 
-# https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
-MACHINES_TO_ZIP = OrderedDict([
-    ("x86_64", "Linux-x64"),
-    ("aarch64", "LinuxArm64"),
-    ("armv8", "LinuxArm64"),
-    ("arm", "LinuxArm"),
-    ("AMD64", "Windows-x64")
-])
-
-def getZipName():
-    for possibleMachine, possibleZipName in MACHINES_TO_ZIP.items():
-        if possibleMachine in machine:
-            return possibleZipName
+def isARM():
+    return True if machine.startswith('arm') else False
 
 def getLatestFileInRepo(repo, search_string):
     # Create an unauthenticated instance of the Github object
@@ -59,11 +46,8 @@ def getLatestFileInRepo(repo, search_string):
         print(f'No release found with {search_string}')
 
 def getLatestCLIRelease():
-    zipName = getZipName()
-    if not zipName:
-        print(f"GetTwitchDownloader.py could not get valid path for '{machine}'. Exiting...")
-        sys.exit(1)
-    searchString = r'.*CLI.*' + zipName
+    isArm = isARM()
+    searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
     getLatestFileInRepo("lay295/TwitchDownloader", searchString)
 
 getLatestCLIRelease()

docker-utils/fetch-twitchdownloader.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
-# and also optimizing some code with this commit.
-# xoxo :D
-
-case $(uname -m) in
-  x86_64)
-    ARCH=Linux-x64;;
-  aarch64)
-    ARCH=LinuxArm64;;
-  armhf)
-    ARCH=LinuxArm;;
-  armv7)
-    ARCH=LinuxArm;;
-  armv7l)
-    ARCH=LinuxArm;;
-  *)
-    echo "Unsupported architecture: $(uname -m)"
-    exit 1
-esac
-
-echo "(INFO) Architecture detected: $ARCH"
-echo "(1/5) READY - Install unzip"
-apt-get update && apt-get -y install unzip curl jq libicu70
-VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases" | jq -r --arg arch "$ARCH" '[.[] | select(.assets | length > 0) | select(.assets[].name | contains("CLI") and contains($arch))] | max_by(.published_at) | .tag_name')
-echo "(2/5) DOWNLOAD - Acquire twitchdownloader"
-curl -o twitchdownloader.zip \
-    --connect-timeout 5 \
-    --max-time 120 \
-    --retry 5 \
-    --retry-delay 0 \
-    --retry-max-time 40 \
-    -L "https://github.com/lay295/TwitchDownloader/releases/download/$VERSION/TwitchDownloaderCLI-$VERSION-$ARCH.zip"
-unzip twitchdownloader.zip
-chmod +x TwitchDownloaderCLI
-echo "(3/5) Smoke test"
-./TwitchDownloaderCLI --help
-cp ./TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI

docker-utils/ffmpeg-fetch.sh

@@ -30,7 +30,7 @@ curl -o ffmpeg.txz \
     --retry 5 \
     --retry-delay 0 \
     --retry-max-time 40 \
-    "https://johnvansickle.com/ffmpeg/old-releases/ffmpeg-5.1.1-${ARCH}-static.tar.xz"
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
 echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.3.2",
+  "version": "4.3.1",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -34,17 +34,18 @@
     "@angular/platform-browser-dynamic": "^15.0.1",
     "@angular/router": "^15.0.1",
     "@fontsource/material-icons": "^4.5.4",
-    "@ngneat/content-loader": "^7.0.0",
+    "@ngneat/content-loader": "^5.0.0",
     "@videogular/ngx-videogular": "^6.0.0",
     "core-js": "^2.4.1",
     "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
     "filesize": "^10.0.7",
+    "fingerprintjs2": "^2.1.0",
     "fs-extra": "^10.0.0",
     "material-icons": "^1.10.8",
     "nan": "^2.14.1",
     "ngx-avatars": "^1.4.1",
-    "ngx-file-drop": "^15.0.0",
+    "ngx-file-drop": "^13.0.0",
     "rxjs": "^6.6.3",
     "rxjs-compat": "^6.6.7",
     "tslib": "^2.0.0",
@@ -65,6 +66,7 @@
     "@typescript-eslint/parser": "^4.29.0",
     "ajv": "^7.2.4",
     "codelyzer": "^6.0.0",
+    "electron": "^19.1.9",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
@@ -78,11 +80,5 @@
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"
-  },
-  "overrides": {
-    "ngx-avatars": {
-      "@angular/common": "15.0.1",
-      "@angular/core": "15.0.1"
-    }
-}
+  }
 }

src/api-types/index.ts

@@ -14,7 +14,6 @@ export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermission
 export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
 export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
 export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
-export type { CheckSubscriptionRequest } from './models/CheckSubscriptionRequest';
 export type { ClearDownloadsRequest } from './models/ClearDownloadsRequest';
 export type { ConcurrentStream } from './models/ConcurrentStream';
 export type { Config } from './models/Config';
@@ -105,7 +104,6 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
 export type { SuccessObject } from './models/SuccessObject';
 export type { TableInfo } from './models/TableInfo';
 export type { Task } from './models/Task';
-export { TaskType } from './models/TaskType';
 export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
 export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
 export type { TransferDBRequest } from './models/TransferDBRequest';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -5,4 +5,4 @@
 export type AddFileToPlaylistRequest = {
     file_uid: string;
     playlist_id: string;
-};
+};

src/api-types/models/Archive.ts

@@ -13,4 +13,4 @@ export type Archive = {
     sub_id?: string;
     timestamp: number;
     uid: string;
-};
+};

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
 export type BaseChangePermissionsRequest = {
     permission: UserPermission;
     new_value: YesNo;
-};
+};

src/api-types/models/Category.ts

@@ -12,4 +12,4 @@ export type Category = {
      * Overrides file output for downloaded files in category
      */
     custom_output?: string;
-};
+};

src/api-types/models/CategoryRule.ts

@@ -22,4 +22,4 @@ export namespace CategoryRule {
     }
 
 
-}
+}

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
 
 export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
 role: string;
-});
+});

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
 
 export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
 user_uid: string;
-});
+});

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
      * UID of the concurrent stream
      */
     uid: string;
-};
+};

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
 
 export type CheckConcurrentStreamResponse = {
     stream: ConcurrentStream;
-};
+};

src/api-types/models/CheckSubscriptionRequest.ts

@@ -1,7 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-export type CheckSubscriptionRequest = {
-    sub_id: string;
-};

src/api-types/models/ClearDownloadsRequest.ts

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
     clear_finished?: boolean;
     clear_paused?: boolean;
     clear_errors?: boolean;
-};
+};

src/api-types/models/ConcurrentStream.ts

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
     playback_timestamp?: number;
     unix_timestamp?: number;
     playing?: boolean;
-};
+};

src/api-types/models/Config.ts

@@ -4,4 +4,4 @@
 
 export type Config = {
     YoutubeDLMaterial: any;
-};
+};

src/api-types/models/ConfigResponse.ts

@@ -7,4 +7,4 @@ import type { Config } from './Config';
 export type ConfigResponse = {
     config_file: Config;
     success: boolean;
-};
+};

src/api-types/models/CreateCategoryRequest.ts

@@ -4,4 +4,4 @@
 
 export type CreateCategoryRequest = {
     name: string;
-};
+};

src/api-types/models/CreateCategoryResponse.ts

@@ -7,4 +7,4 @@ import type { Category } from './Category';
 export type CreateCategoryResponse = {
     new_category?: Category;
     success?: boolean;
-};
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
     playlistName: string;
     uids: Array<string>;
     thumbnailURL: string;
-};
+};

src/api-types/models/CreatePlaylistResponse.ts

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
 export type CreatePlaylistResponse = {
     new_playlist: Playlist;
     success: boolean;
-};
+};

src/api-types/models/CropFileSettings.ts

@@ -5,4 +5,4 @@
 export type CropFileSettings = {
     cropFileStart: number;
     cropFileEnd: number;
-};
+};

src/api-types/models/DBBackup.ts

@@ -17,4 +17,4 @@ export namespace DBBackup {
     }
 
 
-}
+}

src/api-types/models/DBInfoResponse.ts

@@ -16,4 +16,4 @@ roles?: TableInfo;
 download_queue?: TableInfo;
 archives?: TableInfo;
 };
-};
+};

src/api-types/models/DatabaseFile.ts

@@ -42,4 +42,4 @@ export type DatabaseFile = {
      */
     abr?: number;
     favorite: boolean;
-};
+};

src/api-types/models/DeleteAllFilesResponse.ts

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
      * Number of files removed
      */
     delete_count?: number;
-};
+};

src/api-types/models/DeleteArchiveItemsRequest.ts

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
 
 export type DeleteArchiveItemsRequest = {
     archives: Array<Archive>;
-};
+};

src/api-types/models/DeleteCategoryRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeleteCategoryRequest = {
     category_uid: string;
-};
+};

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -5,4 +5,4 @@
 export type DeleteMp3Mp4Request = {
     uid: string;
     blacklistMode?: boolean;
-};
+};

src/api-types/models/DeleteNotificationRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeleteNotificationRequest = {
     uid: string;
-};
+};

src/api-types/models/DeletePlaylistRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeletePlaylistRequest = {
     playlist_id: string;
-};
+};

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -8,4 +8,4 @@ export type DeleteSubscriptionFileRequest = {
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */
     deleteForever?: boolean;
-};
+};

src/api-types/models/DeleteUserRequest.ts

@@ -4,4 +4,4 @@
 
 export type DeleteUserRequest = {
     uid: string;
-};
+};

src/api-types/models/Download.ts

@@ -8,7 +8,6 @@ export type Download = {
     running: boolean;
     finished: boolean;
     paused: boolean;
-    cancelled?: boolean;
     finished_step: boolean;
     url: string;
     type: string;
@@ -28,4 +27,4 @@ export type Download = {
     sub_id?: string;
     sub_name?: string;
     prefetched_info?: any;
-};
+};

src/api-types/models/DownloadArchiveRequest.ts

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
 export type DownloadArchiveRequest = {
     type?: FileType;
     sub_id?: string;
-};
+};

src/api-types/models/DownloadFileRequest.ts

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
     playlist_id?: string;
     url?: string;
     type?: FileType;
-};
+};

src/api-types/models/DownloadRequest.ts

@@ -49,4 +49,4 @@ export type DownloadRequest = {
      * If using youtube-dl archive, download will ignore it
      */
     ignoreArchive?: boolean;
-};
+};

src/api-types/models/DownloadResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type DownloadResponse = {
     download?: Download;
-};
+};

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
      */
     uuid?: string;
     sub?: Subscription;
-};
+};

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
 
 export type DownloadTwitchChatByVODIDResponse = {
     chat: Array<TwitchChatMessage>;
-};
+};

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -4,4 +4,4 @@
 
 export type DownloadVideosForSubscriptionRequest = {
     subID: string;
-};
+};

src/api-types/models/FileType.ts

@@ -5,4 +5,4 @@
 export enum FileType {
     AUDIO = 'audio',
     VIDEO = 'video',
-}
+}

src/api-types/models/FileTypeFilter.ts

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
     AUDIO_ONLY = 'audio_only',
     VIDEO_ONLY = 'video_only',
     BOTH = 'both',
-}
+}

src/api-types/models/GenerateArgsResponse.ts

@@ -4,4 +4,4 @@
 
 export type GenerateArgsResponse = {
     args?: Array<string>;
-};
+};

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -4,4 +4,4 @@
 
 export type GenerateNewApiKeyResponse = {
     new_api_key: string;
-};
+};

src/api-types/models/GetAllCategoriesResponse.ts

@@ -6,4 +6,4 @@ import type { Category } from './Category';
 
 export type GetAllCategoriesResponse = {
     categories: Array<Category>;
-};
+};

src/api-types/models/GetAllDownloadsRequest.ts

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
      * Filters downloads with the array
      */
     uids?: Array<string> | null;
-};
+};

src/api-types/models/GetAllDownloadsResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type GetAllDownloadsResponse = {
     downloads?: Array<Download>;
-};
+};

src/api-types/models/GetAllFilesRequest.ts

@@ -21,4 +21,4 @@ export type GetAllFilesRequest = {
      * Include if you want to filter by subscription
      */
     sub_id?: string;
-};
+};

src/api-types/models/GetAllFilesResponse.ts

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
      * All video playlists
      */
     playlists: Array<Playlist>;
-};
+};

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
 
 export type GetAllSubscriptionsResponse = {
     subscriptions: Array<Subscription>;
-};
+};

src/api-types/models/GetAllTasksResponse.ts

@@ -6,4 +6,4 @@ import type { Task } from './Task';
 
 export type GetAllTasksResponse = {
     tasks?: Array<Task>;
-};
+};

src/api-types/models/GetArchivesRequest.ts

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
 export type GetArchivesRequest = {
     type?: FileType;
     sub_id?: string;
-};
+};

src/api-types/models/GetArchivesResponse.ts

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
 
 export type GetArchivesResponse = {
     archives: Array<Archive>;
-};
+};

src/api-types/models/GetDBBackupsResponse.ts

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
 
 export type GetDBBackupsResponse = {
     tasks?: Array<DBBackup>;
-};
+};

src/api-types/models/GetDownloadRequest.ts

@@ -4,4 +4,4 @@
 
 export type GetDownloadRequest = {
     download_uid: string;
-};
+};

src/api-types/models/GetDownloadResponse.ts

@@ -6,4 +6,4 @@ import type { Download } from './Download';
 
 export type GetDownloadResponse = {
     download?: Download;
-};
+};

src/api-types/models/GetFileFormatsRequest.ts

@@ -4,4 +4,4 @@
 
 export type GetFileFormatsRequest = {
     url?: string;
-};
+};

src/api-types/models/GetFileFormatsResponse.ts

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
     result: {
 formats?: Array<any>;
 };
-};
+};