Added translations for manage-role component

All routes are now properly protected against logged in users w/o permissions

.dockerignore

@@ -0,0 +1,7 @@
+.git
+db
+appdata
+audio
+video
+subscriptions
+users

.eslintrc.json

@@ -0,0 +1,20 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+    }
+}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment**
+- YoutubeDL-Material version
+- Docker tag: <tag> (optional)
+
+Ideally you'd copy the info as presented on the "About" dialogue
+in YoutubeDL-Material.
+(for that, click on the three dots on the top right and then
+check "installation details". On later versions of YoutubeDL-
+Material you will find pretty much all the crucial information
+here that we need in most cases!)
+
+**Additional context**
+Add any other context about the problem here. For example, a YouTube link.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[FEATURE]"
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yaml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/backend/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -0,0 +1,111 @@
+name: continuous integration
+
+on:
+  push:
+    branches: [master, feat/*]
+    tags:
+      - v*
+  pull_request:
+      branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          cache: 'npm'
+      - name: install dependencies
+        run: |
+          npm install
+          cd backend
+          npm install
+          sudo npm install -g @angular/cli
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: build
+        run: npm run build
+      - name: prepare artifact upload
+        shell: pwsh
+        run: |
+          New-Item -Name build -ItemType Directory
+          New-Item -Path build -Name youtubedl-material -ItemType Directory
+          Copy-Item -Path ./backend/appdata -Recurse -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/audio -Recurse -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/authentication -Recurse -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
+          New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
+          Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
+          Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
+      - name: upload build artifact
+        uses: actions/upload-artifact@v1
+        with:
+          name: youtubedl-material
+          path: build
+  release:
+    runs-on: ubuntu-latest
+    needs: build
+    if: contains(github.ref, '/tags/v')
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: YoutubeDL-Material ${{ github.ref }}
+          body: |
+            # New features
+            # Minor additions
+            # Bug fixes
+          draft: true
+          prerelease: false
+      - name: download build artifact
+        uses: actions/download-artifact@v1
+        with:
+          name: youtubedl-material
+          path: ${{runner.temp}}/youtubedl-material
+      - name: extract tag name
+        id: tag_name
+        run: echo ::set-output name=tag_name::${GITHUB_REF#refs/tags/}
+      - name: prepare release asset
+        shell: pwsh
+        run: Compress-Archive -Path ${{runner.temp}}/youtubedl-material -DestinationPath youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
+      - name: upload release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
+          asset_name: youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
+          asset_content_type: application/zip
+      - name: upload docker-compose asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./docker-compose.yml
+          asset_name: docker-compose.yml
+          asset_content_type: text/plain

.github/workflows/close-issue-if-noresponse.yml

@@ -0,0 +1,38 @@
+name: No Response
+
+# Both `issue_comment` and `scheduled` event types are required for this Action
+# to work properly.
+on:
+  issue_comment:
+    types: [created]
+  schedule:
+    # Schedule for five minutes after the hour, every hour
+    - cron: '5 * * * *'
+
+# By specifying the access of one of the scopes, all of those that are not
+# specified are set to 'none'.
+permissions:
+  issues: write
+
+jobs:
+  noResponse:
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'Tzahi12345/YoutubeDL-Material' }}
+    steps:
+      - uses: lee-dohm/no-response@v0.5.0
+        with:
+          token: ${{ github.token }}
+          # Comment to post when closing an Issue for lack of response. Set to `false` to disable
+          closeComment: >
+            This issue has been automatically closed because there has been no response
+            to our request for more information from the original author. With only the
+            information that is currently in the issue, we don't have enough information
+            to take action. Please reach out if you have or find the answers we need so
+            that we can investigate further. We will re-open this issue if you provide us
+            with the requested information with a comment under this issue.
+            Thank you for your understanding and for trying to help make this application
+            a better one!
+          # Number of days of inactivity before an issue is closed for lack of response.
+          daysUntilClose: 21
+          # Label requiring a response.
+          responseRequiredLabel: "💬 response-needed"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 12 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['javascript']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/docker-pr.yml

@@ -0,0 +1,27 @@
+name: docker-pr
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: Build docker images
+        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker-release.yml

@@ -0,0 +1,86 @@
+name: docker-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tags:
+        description: 'Docker tags'
+        required: true
+  release:
+    types: [published]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+
+      - name: Set image tag
+        id: tags
+        run: |
+          if [ "${{ github.event.inputs.tags }}" != "" ]; then
+            echo "::set-output name=tags::${{ github.event.inputs.tags }}"
+          elif [ ${{ github.event.action }} == "release" ]; then
+            echo "::set-output name=tags::${{ github.event.release.tag_name }}"
+          else
+            echo "Unknown workflow trigger: ${{ github.event.action }}! Cannot determine default tag."
+            exit 1
+          fi
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,value=${{ steps.tags.outputs.tags }}
+            type=raw,value=latest
+
+      - name: setup platform emulator
+        uses: docker/setup-qemu-action@v1
+
+      - name: setup multi-arch docker build
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build & push images
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
+          push: true
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/docker.yml

@@ -0,0 +1,86 @@
+name: docker
+
+on:
+  push:
+    branches: [master]
+    paths-ignore:
+      - '.github/**'
+      - '.vscode/**'
+      - 'chrome-extension/**'
+      - 'releases/**'
+      - '**/**.md'
+      - '**.crx'
+      - '**.pem'
+      - '.dockerignore'
+      - '.gitignore'
+  schedule:
+    - cron:  '34 4 * * 2'
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+
+      - name: setup platform emulator
+        uses: docker/setup-qemu-action@v1
+
+      - name: setup multi-arch docker build
+        uses: docker/setup-buildx-action@v1
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        # Defaults:
+        #     DOCKERHUB_USERNAME  : tzahi12345
+        #     DOCKERHUB_REPO      : youtubedl-material
+        #     DOCKERHUB_MASTER_TAG: nightly
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}-{{ date 'YYYY-MM-DD' }}
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}
+            type=sha,prefix=sha-,format=short
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build & push images
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
+          push: true
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}

.gitignore

@@ -25,6 +25,7 @@
 !.vscode/extensions.json
 
 # misc
+/.angular/cache
 /.sass-cache
 /connect.lock
 /coverage
@@ -44,7 +45,34 @@ Thumbs.db
 node_modules/*
 backend/node_modules/*
 YoutubeDL-Material/node_modules/*
-backend/video/*
-backend/audio/*
+backend/subscriptions/archives/*
+backend/subscriptions/playlists/*
+backend/subscriptions/channels/*
 backend/db.json
+backend/public/assets/i18n/*.xlf
+backend/public/assets/default.json
+backend/subscriptions/channels/*
+backend/subscriptions/playlists/*
+backend/subscriptions/archives/*
+backend/*.exe
 src/assets/default.json
+backend/appdata/db.json
+backend/appdata/archives/archive_audio.txt
+backend/appdata/archives/archive_video.txt
+backend/appdata/archives/blacklist_audio.txt
+backend/appdata/archives/blacklist_video.txt
+backend/appdata/logs/combined.log
+backend/appdata/logs/error.log
+backend/appdata/users.json
+backend/users/*
+backend/appdata/cookies.txt
+backend/public
+src/assets/i18n/*.json
+
+# User Files
+db/
+appdata/
+audio/
+video/
+subscriptions/
+users/

.vscode/tasks.json

@@ -0,0 +1,25 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "npm",
+			"script": "start",
+			"problemMatcher": [],
+			"label": "Dev: start frontend",
+			"detail": "ng serve"
+		},
+		{
+			"label": "Dev: start backend",
+			"type": "shell",
+			"command": "set YTDL_MODE=debug && node app.js",
+			"options": {
+				"cwd": "./backend"
+			},
+			"presentation": {
+				"reveal": "always",
+				"panel": "new"
+			},
+			"problemMatcher": []
+		}
+	]
+}

Dockerfile

@@ -1,28 +1,68 @@
-FROM ubuntu:18.04
+# Fetching our ffmpeg
+FROM ubuntu:22.04 AS ffmpeg
+ENV DEBIAN_FRONTEND=noninteractive
+# Use script due local build compability
+COPY ffmpeg-fetch.sh .
+RUN sh ./ffmpeg-fetch.sh
 
-RUN apt-get update && apt-get install -y \
-  nodejs \
-  apache2 \
-  npm \
-  youtube-dl
 
-# Change directory so that our commands run inside this new directory
-WORKDIR /var/www/html
+# Create our Ubuntu 22.04 with node 16
+# Go to 20.04
+FROM ubuntu:20.04 AS base
+ENV DEBIAN_FRONTEND=noninteractive
+ENV UID=1000
+ENV GID=1000
+ENV USER=youtube
+ENV NO_UPDATE_NOTIFIER=true
+ENV PM2_HOME=/app/pm2
+RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
+    apt update && \
+    apt install -y --no-install-recommends curl ca-certificates && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt install -y --no-install-recommends nodejs && \
+    npm -g install npm && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# Copy dependency definitions
-COPY ./ /var/www/html/
 
-# Change directory to backend
-WORKDIR /var/www/html/backend
+# Build frontend
+FROM base as frontend
+RUN npm install -g @angular/cli
+WORKDIR /build
+COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
+COPY [ "src/", "/build/src/" ]
+RUN npm install && \
+    npm run build && \
+    ls -al /build/backend/public
 
-# Install dependencies on backend
-RUN npm install
 
-# Change back to original directory
-WORKDIR /var/www/html
+# Install backend deps
+FROM base as backend
+WORKDIR /app
+COPY [ "backend/","/app/" ]
+RUN npm config set strict-ssl false && \
+    npm install --prod && \
+    ls -al
 
-# Expose the port the app runs in
-EXPOSE 80
 
-# Run the specified command within the container.
-CMD ./docker_wrapper.sh
+# Final image
+FROM base
+RUN npm install -g pm2 && \
+    apt update && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install tcd
+WORKDIR /app
+# User 1000 already exist from base image
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
+COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
+COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
+COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
+RUN chmod +x /app/fix-scripts/*.sh
+# Add some persistence data
+#VOLUME ["/app/appdata"]
+
+EXPOSE 17442
+ENTRYPOINT [ "/app/entrypoint.sh" ]
+CMD [ "npm","start" ]

Dockerfile.heroku

@@ -0,0 +1,2 @@
+FROM tzahi12345/youtubedl-material:latest
+CMD [ "npm", "start" ]

Procfile

@@ -1 +0,0 @@
-web: cd backend && node app.js

README.md

@@ -1,106 +1,156 @@
 # YoutubeDL-Material
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 8](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+[![Docker pulls badge](https://img.shields.io/docker/pulls/tzahi12345/youtubedl-material.svg)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
+[![Docker image size badge](https://img.shields.io/docker/image-size/tzahi12345/youtubedl-material?sort=date)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
+[![Heroku deploy badge](https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg)](https://heroku.com/deploy?template=https://github.com/Tzahi12345/YoutubeDL-Material)
+[![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
+[![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
+
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
+<hr>
+
 ## Getting Started
 
 Check out the prerequisites, and go to the installation section. Easy as pie!
 
 Here's an image of what it'll look like once you're done:
 
-![frontpage](https://i.imgur.com/rOxWIys.png)
-
-With optional file management enabled (default):
-
-![frontpage_with_files](https://i.imgur.com/UTUROLl.png)
+<img src="https://i.imgur.com/C6vFGbL.png" width="800">
 
 Dark mode:
 
-![dark_mode](https://i.imgur.com/9TMkHF6.png?1)
+<img src="https://i.imgur.com/vOtvH5w.png" width="800">
 
 ### Prerequisites
 
-NOTE: If you would like to use Docker, you can go down to the [Docker](#Docker) section for a setup guide.
+NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
 
-You need to have a functioning web server for this to work. Also make sure you have these dependencies installed on your system: nodejs and youtube-dl. If you don't, run this command:
+Debian/Ubuntu:
 
+```bash
+sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
 ```
-sudo apt-get install nodejs youtube-dl
+
+CentOS 7:
+
+```bash
+sudo yum install epel-release
+sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm
+sudo yum install centos-release-scl-rh
+sudo yum install rh-nodejs12
+scl enable rh-nodejs12 bash
+sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 ```
 
+Optional dependencies:
+
+* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
+
 ### Installing
 
-First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
+1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
 
-Drag all the files in `youtubedl-material` to a location accessible to a web server. It works best if it's the root (usually right inside `public_html`. Once that's done, navigate to `backend` and edit the `default.json` file. If you're using SSL encryption, look at the `encrypted.json` file for a template. 
+2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.
 
-Port forward `17442` if you're going to access YoutubeDL-Material from out of your network. This is an important step. Make sure the configuration reflects this appropriately.
+NOTE: If you are intending to use a [reverse proxy](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Reverse-Proxy-Setup), this next step is not necessary
 
-Once the configuration is done, run `npm install` to install all the backend dependencies. Once that is finished, type `nodejs app.js`. This will run the backend server. On your browser, navigate to your installation folder. Try putting in a youtube link to see if it works. If it does, viola! YoutubeDL-Material is now up and running.
+3. Port forward the port listed in `default.json`, which defaults to `17442`.
+
+4. Once the configuration is done, run `npm install` to install all the backend dependencies. Once that is finished, type `npm start`. This will run the backend server, which serves the frontend as well. On your browser, navigate to to the server (url with the specified port). Try putting in a youtube link to see if it works. If it does, viola! YoutubeDL-Material is now up and running.
 
 If you experience problems, know that it's usually caused by a configuration problem. The first thing you should do is check the console. To get there, right click anywhere on the page and click "Inspect element." Then on the menu that pops up, click console. Look at the error there, and try to investigate.
 
-### Configuration
-
-Here is an explanation for the configuration entries. Check out the [default config](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/backend/config/default.json) for more context.
-
-| Config item | Description | Default |
-| ------------- | ------------- | ------------- |
-| frontendurl  | URL to the webserver hosting YTDL-Material | "http://example.com" |
-| backendurl  | URL to the YTDL-Material's backend, should include port 17442 | "http://example.com:17442/" |
-| use-encryption | true if you intend to use SSL encryption (https) | false |
-| cert-file-path | Cert file path - required if using encryption | "/etc/letsencrypt/live/example.com/fullchain.pem" |
-| key-file-path | Private key file path - required if using encryption |  "/etc/letsencrypt/live/example.com/privkey.pem" |
-| path-base | Audio/video stream URL. Usually the same as backendurl | "http://example.com:17442/" |
-| path-audio | Path to audio folder for saved mp3s | "audio/" |
-| path-video | Path to video folder for saved mp4s | "video/" |
-| title_top | Title shown on the top toolbar | "Youtube Downloader" |
-| file_manager_enabled | true if you want to use the file manager | true |
-| allow_quality_select | true if you want to select a videos quality level before downloading | true |
-| download_only_mode | true if you want files to directly download to the client with no media player | false |
-| use_youtube_API | true if you want to use the Youtube API which is used for YT searches | false |
-| youtube_API_key | Youtube API key. Required if use_youtube_API is enabled | "" |
-| default_theme | Default theme to use. Options are "default" and "dark" | "default" |
-| allow_theme_change | true if you want the icon in the top toolbar that toggles dark mode | true |
-
-## Deployment
+## Build it yourself
 
 If you'd like to install YoutubeDL-Material, go to the Installation section. If you want to build it yourself and/or develop the repository, then this section is for you.
 
 To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
 
-Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/backend/config`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/dist` folder. Drag those files into a web server, and drag the `backend` directory into the same folder. This folder should have `index.html` in it as well. If it does **not**, you're in the wrong directory.
+Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
 
-The frontend is now complete. The backend is much easier. Just go into the `youtubedl-material/backend` folder, and type `sudo nodejs app.js`.
+The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
 
-Finally, port forward the port `17442` and point it to the server's IP address. Make sure the port is also allowed through the firewall.
+Finally, if you want your instance to be available from outside your network, you can set up a [reverse proxy](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Reverse-Proxy-Setup).
+
+Alternatively, you can port forward the port specified in the config (defaults to `17442`) and point it to the server's IP address. Make sure the port is also allowed through the server's firewall.
 
 ## Docker
 
+### Host-specific instructions
+
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+
+### Setup
+
 If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
 
-1. Run `curl https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest/download/docker-compose.yml -o docker-compose.yml` to download the latest release of `docker-compose.yml`, or go to the [releases](https://github.com/Tzahi12345/YoutubeDL-Material/releases/) page to grab the version you'd like.
-2. Modify the config items in the `environment` section of `docker-compose.yml` to your liking. Otherwise, the default options will work and point to `http://localhost:8998`. You can find an explanation of these configuration items in [Configuration](#Configuration) section. 
-3. Make sure the port in the `frontend_url` environment variable lines up with the port in the `ports` section.
-4. Run `docker-compose pull`. This will download the official YoutubeDL-Material docker image.
-5. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17442" or something similar. Make sure you can connect to the frontend, and if so, you are done!
+1. Run `curl -L https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest/download/docker-compose.yml -o docker-compose.yml` to download the latest Docker Compose, or go to the [releases](https://github.com/Tzahi12345/YoutubeDL-Material/releases/) page to grab the version you'd like.
+2. Run `docker-compose pull`. This will download the official YoutubeDL-Material docker image.
+3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
+4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
+
+### Custom UID/GID
+
+By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
+
+```yml
+environment:
+    UID: YOUR_UID
+    GID: YOUR_GID
+```
+
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
+## API
+
+[API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)
+
+To get started, go to the settings menu and enable the public API from the *Extra* tab. You can generate an API key if one is missing.
+
+Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above.
+
+## iOS Shortcut 
+
+If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?)
+
+You can download Shortcut [here.](https://routinehub.co/shortcut/10283/)
 
 ## Contributing
 
-Feel free to submit a pull request! I have no guidelines as of yet, so no need to worry about that.
+If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away.
+
+Pull requests are always appreciated! If you're a bit rusty with coding, that's no problem: we can always help you learn. And if that's too scary, that's OK too! You can create issues for features you'd like to see or bugs you encounter, it all helps this project grow.
+
+If you're interested in translating the app into a new language, check out the [Translate](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Translate) wiki page.
 
 ## Authors
 
 * **Isaac Grynsztein** (me!) - *Initial work*
 
-See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
+Official translators:
+
+* Spanish - tzahi12345
+* German - UnlimitedCookies
+* Chinese - TyRoyal
+
+See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
 
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details
 
+## Legal Disclaimer
+
+This project is in no way affiliated with Google LLC, Alphabet Inc. or YouTube (or their subsidiaries) nor endorsed by them.
+
 ## Acknowledgments
 
 * youtube-dl

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+If you would like to see the latest updates, use the `nightly` tag on Docker.
+
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
+
+| Version              | Supported          |
+| -------------------- | ------------------ |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
+
+## Reporting a Vulnerability
+
+Please file an issue in our GitHub's repo, because this app
+isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
+
+We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

angular.json

@@ -7,11 +7,17 @@
       "root": "",
       "sourceRoot": "src",
       "projectType": "application",
+      "i18n": {
+        "sourceLocale": "en-US",
+        "locales": {
+          "es": "src/locale/messages.es.xlf"
+        }
+      },
       "architect": {
         "build": {
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
-            "outputPath": "dist",
+            "outputPath": "backend/public",
             "index": "src/index.html",
             "main": "src/main.ts",
             "tsConfig": "src/tsconfig.app.json",
@@ -24,18 +30,32 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
-            "scripts": []
+            "scripts": [],
+            "vendorChunk": true,
+            "extractLicenses": false,
+            "buildOptimizer": false,
+            "sourceMap": true,
+            "optimization": false,
+            "namedChunks": true,
+            "allowedCommonJsDependencies": [
+              "rxjs",
+              "crypto-js"
+            ]
           },
           "configurations": {
             "production": {
+              "budgets": [
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "6kb"
+                }
+              ],
               "optimization": true,
               "outputHashing": "all",
-              "sourceMap": false,
-              "extractCss": true,
               "namedChunks": false,
-              "aot": true,
               "extractLicenses": true,
               "vendorChunk": false,
               "buildOptimizer": true,
@@ -45,8 +65,12 @@
                   "with": "src/environments/environment.prod.ts"
                 }
               ]
+            },
+            "es": {
+              "localize": ["es"]
             }
-          }
+          },
+          "defaultConfiguration": ""
         },
         "serve": {
           "builder": "@angular-devkit/build-angular:dev-server",
@@ -56,6 +80,9 @@
           "configurations": {
             "production": {
               "browserTarget": "youtube-dl-material:build:production"
+            },
+            "es": {
+              "browserTarget": "youtube-dl-material:build:es"
             }
           }
         },
@@ -92,7 +119,8 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "scripts": []
           },
@@ -125,7 +153,8 @@
             "tsConfig": "src/tsconfig.spec.json",
             "scripts": [],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "assets": [
               "src/assets",
@@ -135,16 +164,6 @@
               "src/backend"
             ]
           }
-        },
-        "lint": {
-          "builder": "@angular-devkit/build-angular:tslint",
-          "options": {
-            "tsConfig": [
-              "src/tsconfig.app.json",
-              "src/tsconfig.spec.json"
-            ],
-            "exclude": []
-          }
         }
       }
     },
@@ -159,15 +178,6 @@
             "protractorConfig": "./protractor.conf.js",
             "devServerTarget": "youtube-dl-material:serve"
           }
-        },
-        "lint": {
-          "builder": "@angular-devkit/build-angular:tslint",
-          "options": {
-            "tsConfig": [
-              "e2e/tsconfig.e2e.json"
-            ],
-            "exclude": []
-          }
         }
       }
     }
@@ -176,7 +186,7 @@
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",
-      "styleext": "scss"
+      "style": "scss"
     },
     "@schematics/angular:directive": {
       "prefix": "app"

app.json

@@ -0,0 +1,8 @@
+{
+    "name": "YoutubeDL-Material",
+    "description": "An open-source and self-hosted YouTube downloader based on Google's Material Design specifications.",
+    "repository": "https://github.com/Tzahi12345/YoutubeDL-Material",
+    "stack": "container",
+    "logo": "https://i.imgur.com/GPzvPiU.png",
+    "keywords": ["youtube-dl", "youtubedl-material", "nodejs"]
+  }

backend/.dockerignore

@@ -0,0 +1,4 @@
+node_modules
+*.exe
+docker-compose.yml
+Dockerfile

backend/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "env": {
+        "node": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended"
+    ],
+    "parser": "esprima",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [],
+    "rules": {
+    },
+    "root": true
+}

backend/appdata/default.json

@@ -0,0 +1,77 @@
+{
+  "YoutubeDLMaterial": {
+    "Host": {
+      "url": "http://example.com",
+      "port": "17442"
+    },
+    "Downloader": {
+      "path-audio": "audio/",
+      "path-video": "video/",
+      "default_file_output": "",
+      "use_youtubedl_archive": false,
+      "custom_args": "",
+      "safe_download_override": false,
+      "include_thumbnail": true,
+      "include_metadata": true,
+      "max_concurrent_downloads": 5,
+      "download_rate_limit": ""
+    },
+    "Extra": {
+      "title_top": "YoutubeDL-Material",
+      "file_manager_enabled": true,
+      "allow_quality_select": true,
+      "download_only_mode": false,
+      "allow_autoplay": true,
+      "enable_downloads_manager": true,
+      "allow_playlist_categorization": true
+    },
+    "API": {
+      "use_API_key": false,
+      "API_key": "",
+      "use_youtube_API": false,
+      "youtube_API_key": "",
+      "use_twitch_API": false,
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
+      "twitch_auto_download_chat": false,
+      "use_sponsorblock_API": false,
+      "generate_NFO_files": false
+    },
+    "Themes": {
+      "default_theme": "default",
+      "allow_theme_change": true
+    },
+    "Subscriptions": {
+      "allow_subscriptions": true,
+      "subscriptions_base_path": "subscriptions/",
+      "subscriptions_check_interval": "300",
+      "redownload_fresh_uploads": false
+    },
+    "Users": {
+      "base_path": "users/",
+      "allow_registration": true,
+      "auth_method": "internal",
+      "ldap_config": {
+        "url": "ldap://localhost:389",
+        "bindDN": "cn=root",
+        "bindCredentials": "secret",
+        "searchBase": "ou=passport-ldapauth",
+        "searchFilter": "(uid={{username}})"
+      }
+    },
+    "Database": {
+      "use_local_db": true,
+      "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
+    },
+    "Advanced": {
+      "default_downloader": "yt-dlp",
+      "use_default_downloading_agent": true,
+      "custom_downloading_agent": "",
+      "multi_user_mode": false,
+      "allow_advanced_download": false,
+      "use_cookies": false,
+      "jwt_expiration": 86400,
+      "logger_level": "info"
+    }
+  }
+}

backend/audio/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

backend/authentication/auth.js

@@ -0,0 +1,463 @@
+const config_api = require('../config');
+const consts = require('../consts');
+const logger = require('../logger');
+const db_api = require('../db');
+
+const jwt = require('jsonwebtoken');
+const { uuid } = require('uuidv4');
+const bcrypt = require('bcryptjs');
+
+var LocalStrategy = require('passport-local').Strategy;
+var LdapStrategy = require('passport-ldapauth');
+var JwtStrategy = require('passport-jwt').Strategy,
+    ExtractJwt = require('passport-jwt').ExtractJwt;
+
+// other required vars
+let SERVER_SECRET = null;
+let JWT_EXPIRATION = null;
+let opts = null;
+let saltRounds = null;
+
+exports.initialize = function () {
+  /*************************
+   * Authentication module
+   ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
+  saltRounds = 10;
+
+  JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
+
+  SERVER_SECRET = null;
+  if (db_api.users_db.get('jwt_secret').value()) {
+    SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
+  } else {
+    SERVER_SECRET = uuid();
+    db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
+  }
+
+  opts = {}
+  opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
+  opts.secretOrKey = SERVER_SECRET;
+
+  exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
+    const user = await db_api.getRecord('users', {uid: jwt_payload.user});
+    if (user) {
+        return done(null, user);
+    } else {
+        return done(null, false);
+        // or you could create a new account
+    }
+  }));
+}
+
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
+exports.passport = require('passport');
+
+exports.passport.serializeUser(function(user, done) {
+    done(null, user);
+});
+
+exports.passport.deserializeUser(function(user, done) {
+    done(null, user);
+});
+
+/***************************************
+ * Register user with hashed password
+ **************************************/
+exports.registerUser = async function(req, res) {
+  var userid = req.body.userid;
+  var username = req.body.username;
+  var plaintextPassword = req.body.password;
+
+  if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
+    res.sendStatus(409);
+    logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
+    return;
+  }
+
+  if (plaintextPassword === "") {
+    res.sendStatus(400);
+    logger.error(`Registration failed for user ${userid}. A password must be provided.`);
+    return;
+  }
+
+  bcrypt.hash(plaintextPassword, saltRounds)
+    .then(async function(hash) {
+      let new_user = generateUserObject(userid, username, hash);
+      // check if user exists
+      if (await db_api.getRecord('users', {uid: userid})) {
+        // user id is taken!
+        logger.error('Registration failed: UID is already taken!');
+        res.status(409).send('UID is already taken!');
+      } else if (await db_api.getRecord('users', {name: username})) {
+          // user name is taken!
+          logger.error('Registration failed: User name is already taken!');
+          res.status(409).send('User name is already taken!');
+      } else {
+        // add to db
+        await db_api.insertRecordIntoTable('users', new_user);
+        logger.verbose(`New user created: ${new_user.name}`);
+        res.send({
+          user: new_user
+        });
+      }
+    })
+    .then(function(result) {
+
+    })
+    .catch(function(err) {
+      logger.error(err);
+      if( err.code == 'ER_DUP_ENTRY' ) {
+        res.status(409).send('UserId already taken');
+      } else {
+        res.sendStatus(409);
+      }
+    });
+}
+
+/***************************************
+ * Login methods
+ **************************************/
+
+/*************************************************
+ * This gets called when passport.authenticate()
+ * gets called.
+ *
+ * This checks that the credentials are valid.
+ * If so, passes the user info to the next middleware.
+ ************************************************/
+
+
+exports.login = async (username, password) => {
+  // even if we're using LDAP, we still want users to be able to login using internal credentials
+  const user = await db_api.getRecord('users', {name: username});
+  if (!user) {
+    if (config_api.getConfigItem('ytdl_auth_method') === 'internal') logger.error(`User ${username} not found`);
+    return false;
+  }
+  if (user.auth_method && user.auth_method !== 'internal') { return false }
+  return await bcrypt.compare(password, user.passhash) ? user : false;
+}
+
+exports.passport.use(new LocalStrategy({
+    usernameField: 'username',
+    passwordField: 'password'},
+    async function(username, password, done) {
+      return done(null, await exports.login(username, password));
+    }
+));
+
+var getLDAPConfiguration = function(req, callback) {
+  const ldap_config = config_api.getConfigItem('ytdl_ldap_config');
+  const opts = {server: ldap_config};
+  callback(null, opts);
+};
+
+exports.passport.use(new LdapStrategy(getLDAPConfiguration,
+  async function(user, done) {
+    // check if ldap auth is enabled
+    const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
+    if (!ldap_enabled) return done(null, false);
+
+    const user_uid = user.uid;
+    let db_user = await db_api.getRecord('users', {uid: user_uid});
+    if (!db_user) {
+      // generate DB user
+      let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
+      await db_api.insertRecordIntoTable('users', new_user);
+      db_user = new_user;
+      logger.verbose(`Generated new user ${user_uid} using LDAP`);
+    }
+    return done(null, db_user);
+  }
+));
+
+
+/**********************************
+ * Generating/Signing a JWT token
+ * And attaches the user info into
+ * the payload to be sent on every
+ * request.
+ *********************************/
+exports.generateJWT = function(req, res, next) {
+  var payload = {
+      exp: Math.floor(Date.now() / 1000) + JWT_EXPIRATION
+    , user: req.user.uid
+  };
+  req.token = jwt.sign(payload, SERVER_SECRET);
+  next();
+}
+
+exports.returnAuthResponse = async function(req, res) {
+  res.status(200).json({
+    user: req.user,
+    token: req.token,
+    permissions: await exports.userPermissions(req.user.uid),
+    available_permissions: consts['AVAILABLE_PERMISSIONS']
+  });
+}
+
+/***************************************
+ * Authorization: middleware that checks the
+ * JWT token for validity before allowing
+ * the user to access anything.
+ *
+ * It also passes the user object to the next
+ * middleware through res.locals
+ **************************************/
+exports.ensureAuthenticatedElseError = (req, res, next) => {
+  var token = getToken(req.query);
+  if( token ) {
+    try {
+      var payload = jwt.verify(token, SERVER_SECRET);
+      // console.log('payload: ' + JSON.stringify(payload));
+      // check if user still exists in database if you'd like
+      res.locals.user = payload.user;
+      next();
+    } catch(err) {
+      res.status(401).send('Invalid Authentication');
+    }
+  } else {
+    res.status(401).send('Missing Authorization header');
+  }
+}
+
+// change password
+exports.changeUserPassword = async (user_uid, new_pass) => {
+  try {
+    const hash = await bcrypt.hash(new_pass, saltRounds);
+    await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
+    return true;
+  } catch (err) {
+    return false;
+  }
+}
+
+// change user permissions
+exports.changeUserPermissions = async (user_uid, permission, new_value) => {
+  try {
+    await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
+    await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
+    if (new_value === 'yes') {
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
+    } else if (new_value === 'no') {
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
+    }
+    return true;
+  } catch (err) {
+    logger.error(err);
+    return false;
+  }
+}
+
+// change role permissions
+exports.changeRolePermissions = async (role, permission, new_value) => {
+  try {
+    await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
+    if (new_value === 'yes') {
+      await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
+    }
+    return true;
+  } catch (err) {
+    logger.error(err);
+    return false;
+  }
+}
+
+exports.adminExists = async function() {
+  return !!(await db_api.getRecord('users', {uid: 'admin'}));
+}
+
+// video stuff
+
+exports.getUserVideos = async function(user_uid, type) {
+    const files = await db_api.getRecords('files', {user_uid: user_uid});
+    return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
+}
+
+exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
+  let file = await db_api.getRecord('files', {file_uid: file_uid});
+
+  // prevent unauthorized users from accessing the file info
+  if (file && !file['sharingEnabled'] && requireSharing) file = null;
+
+  return file;
+}
+
+exports.removePlaylist = async function(user_uid, playlistID) {
+  await db_api.removeRecord('playlist', {playlistID: playlistID});
+  return true;
+}
+
+exports.getUserPlaylists = async function(user_uid) {
+  return await db_api.getRecords('playlists', {user_uid: user_uid});
+}
+
+exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
+  let playlist = await db_api.getRecord('playlists', {id: playlistID});
+
+  // prevent unauthorized users from accessing the file info
+  if (requireSharing && !playlist['sharingEnabled']) playlist = null;
+
+  return playlist;
+}
+
+exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
+  let success = false;
+  is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
+  success = true;
+  return success;
+}
+
+exports.userHasPermission = async function(user_uid, permission) {
+
+  const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
+  const role = user_obj['role'];
+  if (!role) {
+    // role doesn't exist
+    logger.error('Invalid role ' + role);
+    return false;
+  }
+
+  const user_has_explicit_permission = user_obj['permissions'].includes(permission);
+  const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
+
+  // check if user has a negative/positive override
+  if (user_has_explicit_permission && permission_in_overrides) {
+    // positive override
+    return true;
+  } else if (!user_has_explicit_permission && permission_in_overrides) {
+    // negative override
+    return false;
+  }
+
+  // no overrides, let's check if the role has the permission
+  const role_has_permission = await exports.roleHasPermissions(role, permission);
+  if (role_has_permission) {
+    return true;
+  } else {
+    logger.verbose(`User ${user_uid} failed to get permission ${permission}`);
+    return false;
+  }
+}
+
+exports.roleHasPermissions = async function(role, permission) {
+  const role_obj = await db_api.getRecord('roles', {key: role})
+  if (!role) {
+    logger.error(`Role ${role} does not exist!`);
+  }
+  const role_permissions = role_obj['permissions'];
+  if (role_permissions && role_permissions.includes(permission)) return true;
+  else return false;
+}
+
+exports.userPermissions = async function(user_uid) {
+  let user_permissions = [];
+  const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
+  const role = user_obj['role'];
+  if (!role) {
+    // role doesn't exist
+    logger.error('Invalid role ' + role);
+    return null;
+  }
+  const role_obj = await db_api.getRecord('roles', {key: role});
+  const role_permissions = role_obj['permissions'];
+
+  for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
+    let permission = consts['AVAILABLE_PERMISSIONS'][i];
+
+    const user_has_explicit_permission = user_obj['permissions'].includes(permission);
+    const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
+
+    // check if user has a negative/positive override
+    if (user_has_explicit_permission && permission_in_overrides) {
+      // positive override
+      user_permissions.push(permission);
+    } else if (!user_has_explicit_permission && permission_in_overrides) {
+      // negative override
+      continue;
+    }
+
+    // no overrides, let's check if the role has the permission
+    if (role_permissions.includes(permission)) {
+      user_permissions.push(permission);
+    } else {
+      continue;
+    }
+  }
+
+  return user_permissions;
+}
+
+function getToken(queryParams) {
+  if (queryParams && queryParams.jwt) {
+    var parted = queryParams.jwt.split(' ');
+    if (parted.length === 2) {
+      return parted[1];
+    } else {
+      return null;
+    }
+  } else {
+    return null;
+  }
+};
+
+function generateUserObject(userid, username, hash, auth_method = 'internal') {
+  let new_user = {
+    name: username,
+    uid: userid,
+    passhash: auth_method === 'internal' ? hash : null,
+    files: [],
+    playlists: [],
+    subscriptions: [],
+    created: Date.now(),
+    role: userid === 'admin' && auth_method === 'internal' ? 'admin' : 'user',
+    permissions: [],
+    permission_overrides: [],
+    auth_method: auth_method
+  };
+  return new_user;
+}

backend/categories.js

@@ -0,0 +1,136 @@
+const utils = require('./utils');
+const logger = require('./logger');
+const db_api = require('./db');
+/*
+
+Categories:
+
+    Categories are a way to organize videos based on dynamic rules set by the user. Categories are universal (so not per-user).
+    
+    Categories, besides rules, have an optional custom output. This custom output can help users create their
+        desired directory structure.
+
+Rules:
+    A category rule consists of a property, a comparison, and a value. For example, "uploader includes 'VEVO'"
+
+    Rules are stored as an object with the above fields. In addition to those fields, it also has a preceding_operator, which
+        is either OR or AND, and signifies whether the rule should be ANDed with the previous rules, or just ORed. For the first
+        rule, this field is null.
+
+    Ex. (title includes 'Rihanna' OR title includes 'Beyonce' AND uploader includes 'VEVO')
+
+*/
+
+async function categorize(file_jsons) {
+    // to make the logic easier, let's assume the file metadata is an array
+    if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
+
+    let selected_category = null;
+    const categories = await getCategories();
+    if (!categories) {
+        logger.warn('Categories could not be found.');
+        return null;
+    }
+
+    for (let i = 0; i < file_jsons.length; i++) {
+        const file_json = file_jsons[i];
+        for (let j = 0; j < categories.length; j++) {
+            const category = categories[j];
+            const rules = category['rules'];
+    
+            // if rules for current category apply, then that is the selected category
+            if (applyCategoryRules(file_json, rules, category['name'])) {
+                selected_category = category;
+                logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`);
+                return selected_category;
+            }
+        }
+    }
+    
+    return selected_category;
+}
+
+async function getCategories() {
+    const categories = await db_api.getRecords('categories');
+    return categories ? categories : null;
+}
+
+async function getCategoriesAsPlaylists() {
+    const categories_as_playlists = [];
+    const available_categories = await getCategories();
+    if (available_categories) {
+        for (let category of available_categories) {
+            const files_that_match = await db_api.getRecords('files', {'category.uid': category['uid']});
+            if (files_that_match && files_that_match.length > 0) {
+                category['thumbnailURL'] = files_that_match[0].thumbnailURL;
+                category['thumbnailPath'] = files_that_match[0].thumbnailPath;
+                category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
+                category['id'] = category['uid'];
+                category['auto'] = true;
+                categories_as_playlists.push(category);
+            }
+        }
+    }
+    return categories_as_playlists;
+}
+
+function applyCategoryRules(file_json, rules, category_name) {
+    let rules_apply = false;
+    for (let i = 0; i < rules.length; i++) {
+        const rule = rules[i];
+        let rule_applies = null;
+
+        let preceding_operator = rule['preceding_operator'];
+
+        switch (rule['comparator']) {
+            case 'includes':
+                rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
+                break;
+            case 'not_includes':
+                rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
+                break;
+            case 'equals':
+                rule_applies = file_json[rule['property']] === rule['value'];
+                break;
+            case 'not_equals':
+                rule_applies = file_json[rule['property']] !== rule['value'];
+                break;
+            default:
+                logger.warn(`Invalid comparison used for category ${category_name}`)
+                break;
+        }
+
+        // OR the first rule with rules_apply, which will be initially false
+        if (i === 0) preceding_operator = 'or';
+
+        // update rules_apply based on current rule
+        if (preceding_operator === 'or')
+            rules_apply = rules_apply || rule_applies;
+        else
+            rules_apply = rules_apply && rule_applies;
+    }
+
+    return rules_apply;
+}
+
+// async function addTagToVideo(tag, video, user_uid) {
+//     // TODO: Implement
+// }
+
+// async function removeTagFromVideo(tag, video, user_uid) {
+//     // TODO: Implement
+// }
+
+// // adds tag to list of existing tags (used for tag suggestions)
+// async function addTagToExistingTags(tag) {
+//     const existing_tags = db.get('tags').value();
+//     if (!existing_tags.includes(tag)) {
+//         db.get('tags').push(tag).write();
+//     }
+// }
+
+module.exports = {
+    categorize: categorize,
+    getCategories: getCategories,
+    getCategoriesAsPlaylists: getCategoriesAsPlaylists
+}

backend/config.js

@@ -1,8 +1,31 @@
+const logger = require('./logger');
+
 const fs = require('fs');
 
 let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
+const debugMode = process.env.YTDL_MODE === 'debug';
 
-let configPath = 'config/default.json';
+let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
+
+function initialize() {
+    ensureConfigFileExists();
+    ensureConfigItemsExist();
+}
+
+function ensureConfigItemsExist() {
+    const config_keys = Object.keys(CONFIG_ITEMS);
+    for (let i = 0; i < config_keys.length; i++) {
+        const config_key = config_keys[i];
+        getConfigItem(config_key);
+    }
+}
+
+function ensureConfigFileExists() {
+    if (!fs.existsSync(configPath)) {
+        logger.info('Cannot find config file. Creating one with default values...');
+        fs.writeFileSync(configPath, JSON.stringify(DEFAULT_CONFIG, null, 2));
+    }
+}
 
 // https://stackoverflow.com/questions/6491463/accessing-nested-javascript-objects-with-string-key
 Object.byString = function(o, s) {
@@ -31,16 +54,26 @@ function getElementNameInConfig(path) {
     return elements[elements.length - 1];
 }
 
+/**
+ * Check if config exists. If not, write default config to config path
+ */
+function configExistsCheck() {
+    let exists = fs.existsSync(configPath);
+    if (!exists) {
+        setConfigFile(DEFAULT_CONFIG);
+    }
+}
+
 /*
 * Gets config file and returns as a json
 */
 function getConfigFile() {
-    let raw_data = fs.readFileSync(configPath);
     try {
+        let raw_data = fs.readFileSync(configPath);
         let parsed_data = JSON.parse(raw_data);
         return parsed_data;
     } catch(e) {
-        console.log('ERROR: Failed to get config file');
+        logger.error('Failed to get config file');
         return null;
     }
 }
@@ -56,29 +89,45 @@ function setConfigFile(config) {
 
 function getConfigItem(key) {
     let config_json = getConfigFile();
-    if (!CONFIG_ITEMS[key]) console.log('cannot find config with key ' + key);
+    if (!CONFIG_ITEMS[key]) {
+        logger.error(`Config item with key '${key}' is not recognized.`);
+        return null;
+    }
     let path = CONFIG_ITEMS[key]['path'];
+    const val = Object.byString(config_json, path);
+    if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
+        logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
+        setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
+        return Object.byString(DEFAULT_CONFIG, path);
+    }
     return Object.byString(config_json, path);
-};
+}
 
 function setConfigItem(key, value) {
     let success = false;
     let config_json = getConfigFile();
     let path = CONFIG_ITEMS[key]['path'];
-    let parent_path = getParentPath(path);
     let element_name = getElementNameInConfig(path);
-    
+    let parent_path = getParentPath(path);
     let parent_object = Object.byString(config_json, parent_path);
+    if (!parent_object) {
+        let parent_parent_path = getParentPath(parent_path);
+        let parent_parent_object = Object.byString(config_json, parent_parent_path);
+        let parent_path_arr = parent_path.split('.');
+        let parent_parent_single_key = parent_path_arr[parent_path_arr.length-1];
+        parent_parent_object[parent_parent_single_key] = {};
+        parent_object = Object.byString(config_json, parent_path);
+    }
+
     if (value === 'false' || value === 'true') {
         parent_object[element_name] = (value === 'true');
     } else {
         parent_object[element_name] = value;
     }
-
     success = setConfigFile(config_json);
 
     return success;
-};
+}
 
 function setConfigItems(items) {
     let success = false;
@@ -95,7 +144,7 @@ function setConfigItems(items) {
         let item_path = CONFIG_ITEMS[key]['path'];
         let item_parent_path = getParentPath(item_path);
         let item_element_name = getElementNameInConfig(item_path);
-        
+
         let item_parent_object = Object.byString(config_json, item_parent_path);
         item_parent_object[item_element_name] = value;
     }
@@ -104,9 +153,100 @@ function setConfigItems(items) {
     return success;
 }
 
+function globalArgsRequiresSafeDownload() {
+    const globalArgs = getConfigItem('ytdl_custom_args').split(',,');
+    const argsThatRequireSafeDownload = ['--write-sub', '--write-srt', '--proxy'];
+    const failedArgs = globalArgs.filter(arg => argsThatRequireSafeDownload.includes(arg));
+    return failedArgs && failedArgs.length > 0;
+}
+
 module.exports = {
     getConfigItem: getConfigItem,
     setConfigItem: setConfigItem,
     setConfigItems: setConfigItems,
-    CONFIG_ITEMS: CONFIG_ITEMS
-}
+    getConfigFile: getConfigFile,
+    setConfigFile: setConfigFile,
+    configExistsCheck: configExistsCheck,
+    CONFIG_ITEMS: CONFIG_ITEMS,
+    initialize: initialize,
+    descriptors: {},
+    globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
+}
+
+const DEFAULT_CONFIG = {
+    "YoutubeDLMaterial": {
+      "Host": {
+        "url": "http://example.com",
+        "port": "17442"
+      },
+      "Downloader": {
+        "path-audio": "audio/",
+        "path-video": "video/",
+        "default_file_output": "",
+        "use_youtubedl_archive": false,
+        "custom_args": "",
+        "safe_download_override": false,
+        "include_thumbnail": true,
+        "include_metadata": true,
+        "max_concurrent_downloads": 5,
+        "download_rate_limit": ""
+      },
+      "Extra": {
+        "title_top": "YoutubeDL-Material",
+        "file_manager_enabled": true,
+        "allow_quality_select": true,
+        "download_only_mode": false,
+        "allow_autoplay": true,
+        "enable_downloads_manager": true,
+        "allow_playlist_categorization": true
+      },
+      "API": {
+        "use_API_key": false,
+        "API_key": "",
+        "use_youtube_API": false,
+        "youtube_API_key": "",
+        "use_twitch_API": false,
+        "twitch_client_ID": "",
+        "twitch_client_secret": "",
+        "twitch_auto_download_chat": false,
+        "use_sponsorblock_API": false,
+        "generate_NFO_files": false
+      },
+      "Themes": {
+        "default_theme": "default",
+        "allow_theme_change": true
+      },
+      "Subscriptions": {
+        "allow_subscriptions": true,
+        "subscriptions_base_path": "subscriptions/",
+        "subscriptions_check_interval": "86400",
+        "redownload_fresh_uploads": false
+      },
+      "Users": {
+        "base_path": "users/",
+        "allow_registration": true,
+        "auth_method": "internal",
+        "ldap_config": {
+            "url": "ldap://localhost:389",
+            "bindDN": "cn=root",
+            "bindCredentials": "secret",
+            "searchBase": "ou=passport-ldapauth",
+            "searchFilter": "(uid={{username}})"
+        }
+      },
+      "Database": {
+        "use_local_db": true,
+        "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
+      },
+      "Advanced": {
+        "default_downloader": "yt-dlp",
+        "use_default_downloading_agent": true,
+        "custom_downloading_agent": "",
+        "multi_user_mode": false,
+        "allow_advanced_download": false,
+        "use_cookies": false,
+        "jwt_expiration": 86400,
+        "logger_level": "info"
+      }
+    }
+  }

backend/config/default.json

@@ -1,38 +0,0 @@
-{
-    "YoutubeDLMaterial": {
-        "Host": {
-            "frontendurl": "http://example.com",
-            "backendurl": "http://example.com:17442/"
-        },
-      "Encryption": {
-        "use-encryption": false,
-        "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem",
-        "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem"
-      },
-      "Downloader": {
-        "path-base": "http://example.com:17442/",
-        "path-audio": "audio/",
-        "path-video": "video/"
-      },
-      "Extra": {
-        "title_top": "Youtube Downloader",
-        "file_manager_enabled": true,
-        "allow_quality_select": true,
-        "download_only_mode": false,
-        "allow_multi_download_mode": true
-      },
-      "API": {
-        "use_youtube_API": false,
-        "youtube_API_key": ""
-      },
-      "Themes": {
-        "default_theme": "default",
-        "allow_theme_change": true
-      },
-      "Advanced": {
-        "use_default_downloading_agent": true,
-        "custom_downloading_agent": "",
-        "allow_advanced_download": false
-      }
-    }
-  }

backend/config/encrypted.json

@@ -1,38 +0,0 @@
-{
-    "YoutubeDLMaterial": {
-        "Host": {
-            "frontendurl": "https://example.com",
-            "backendurl": "https://example.com:17442/"
-        },
-      "Encryption": {
-        "use-encryption": true,
-        "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem",
-        "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem"
-      },
-      "Downloader": {
-        "path-base": "https://example.com:17442/",
-        "path-audio": "audio/",
-        "path-video": "video/"
-      },
-      "Extra": {
-        "title_top": "Youtube Downloader",
-        "file_manager_enabled": true,
-        "allow_quality_select": true,
-        "download_only_mode": false,
-        "allow_multi_download_mode": true
-      },
-      "API": {
-        "use_youtube_API": false,
-        "youtube_API_key": ""
-      },
-      "Themes": {
-        "default_theme": "default",
-        "allow_theme_change": true
-      },
-      "Advanced": {
-        "use_default_downloading_agent": true,
-        "custom_downloading_agent": "",
-        "allow_advanced_download": false
-      }
-    }
-  }

backend/consts.js

@@ -1,35 +1,15 @@
-var config = require('config');
-
-let CONFIG_ITEMS = {
+exports.CONFIG_ITEMS = {
     // Host
-    'ytdl_frontend_url': {
-        'key': 'ytdl_frontend_url',
-        'path': 'YoutubeDLMaterial.Host.frontendurl'
+    'ytdl_url': {
+        'key': 'ytdl_url',
+        'path': 'YoutubeDLMaterial.Host.url'
     },
-    'ytdl_backend_url': {
-        'key': 'ytdl_backend_url',
-        'path': 'YoutubeDLMaterial.Host.backendurl'
-    },
-
-    // Encryption
-    'ytdl_use_encryption': {
-        'key': 'ytdl_use_encryption',
-        'path': 'YoutubeDLMaterial.Encryption.use-encryption'
-    },
-    'ytdl_cert_file_path': {
-        'key': 'ytdl_cert_file_path',
-        'path': 'YoutubeDLMaterial.Encryption.cert-file-path'
-    },
-    'ytdl_key_file_path': {
-        'key': 'ytdl_key_file_path',
-        'path': 'YoutubeDLMaterial.Encryption.key-file-path'
+    'ytdl_port': {
+        'key': 'ytdl_port',
+        'path': 'YoutubeDLMaterial.Host.port'
     },
 
     // Downloader
-    'ytdl_base_path': {
-        'key': 'ytdl_base_path',
-        'path': 'YoutubeDLMaterial.Downloader.path-base'
-    },
     'ytdl_audio_folder_path': {
         'key': 'ytdl_audio_folder_path',
         'path': 'YoutubeDLMaterial.Downloader.path-audio'
@@ -38,6 +18,38 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_video_folder_path',
         'path': 'YoutubeDLMaterial.Downloader.path-video'
     },
+    'ytdl_default_file_output': {
+        'key': 'ytdl_default_file_output',
+        'path': 'YoutubeDLMaterial.Downloader.default_file_output'
+    },
+    'ytdl_use_youtubedl_archive': {
+        'key': 'ytdl_use_youtubedl_archive',
+        'path': 'YoutubeDLMaterial.Downloader.use_youtubedl_archive'
+    },
+    'ytdl_custom_args': {
+        'key': 'ytdl_custom_args',
+        'path': 'YoutubeDLMaterial.Downloader.custom_args'
+    },
+    'ytdl_safe_download_override': {
+        'key': 'ytdl_safe_download_override',
+        'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
+    },
+    'ytdl_include_thumbnail': {
+        'key': 'ytdl_include_thumbnail',
+        'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
+    },
+    'ytdl_include_metadata': {
+        'key': 'ytdl_include_metadata',
+        'path': 'YoutubeDLMaterial.Downloader.include_metadata'
+    },
+    'ytdl_max_concurrent_downloads': {
+        'key': 'ytdl_max_concurrent_downloads',
+        'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
+    },
+    'ytdl_download_rate_limit': {
+        'key': 'ytdl_download_rate_limit',
+        'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
+    },
 
     // Extra
     'ytdl_title_top': {
@@ -56,13 +68,28 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_multi_download_mode': {
-        'key': 'ytdl_allow_multi_download_mode',
-        'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
+    'ytdl_allow_autoplay': {
+        'key': 'ytdl_allow_autoplay',
+        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+    },
+    'ytdl_enable_downloads_manager': {
+        'key': 'ytdl_enable_downloads_manager',
+        'path': 'YoutubeDLMaterial.Extra.enable_downloads_manager'
+    },
+    'ytdl_allow_playlist_categorization': {
+        'key': 'ytdl_allow_playlist_categorization',
+        'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
     },
-    
 
     // API
+    'ytdl_use_api_key': {
+        'key': 'ytdl_use_api_key',
+        'path': 'YoutubeDLMaterial.API.use_API_key'
+    },
+    'ytdl_api_key': {
+        'key': 'ytdl_api_key',
+        'path': 'YoutubeDLMaterial.API.API_key'
+    },
     'ytdl_use_youtube_api': {
         'key': 'ytdl_use_youtube_api',
         'path': 'YoutubeDLMaterial.API.use_youtube_API'
@@ -71,6 +98,31 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
+    'ytdl_use_twitch_api': {
+        'key': 'ytdl_use_twitch_api',
+        'path': 'YoutubeDLMaterial.API.use_twitch_API'
+    },
+    'ytdl_twitch_client_id': {
+        'key': 'ytdl_twitch_client_id',
+        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
+    },
+    'ytdl_twitch_client_secret': {
+        'key': 'ytdl_twitch_client_secret',
+        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
+    },
+    'ytdl_twitch_auto_download_chat': {
+        'key': 'ytdl_twitch_auto_download_chat',
+        'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
+    },
+    'ytdl_use_sponsorblock_api': {
+        'key': 'ytdl_use_sponsorblock_api',
+        'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
+    },
+    'ytdl_generate_nfo_files': {
+        'key': 'ytdl_generate_nfo_files',
+        'path': 'YoutubeDLMaterial.API.generate_NFO_files'
+    },
+
 
     // Themes
     'ytdl_default_theme': {
@@ -82,7 +134,57 @@ let CONFIG_ITEMS = {
         'path': 'YoutubeDLMaterial.Themes.allow_theme_change'
     },
 
+    // Subscriptions
+    'ytdl_allow_subscriptions': {
+        'key': 'ytdl_allow_subscriptions',
+        'path': 'YoutubeDLMaterial.Subscriptions.allow_subscriptions'
+    },
+    'ytdl_subscriptions_base_path': {
+        'key': 'ytdl_subscriptions_base_path',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_base_path'
+    },
+    'ytdl_subscriptions_check_interval': {
+        'key': 'ytdl_subscriptions_check_interval',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
+    },
+    'ytdl_subscriptions_redownload_fresh_uploads': {
+        'key': 'ytdl_subscriptions_redownload_fresh_uploads',
+        'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
+    },
+
+    // Users
+    'ytdl_users_base_path': {
+        'key': 'ytdl_users_base_path',
+        'path': 'YoutubeDLMaterial.Users.base_path'
+    },
+    'ytdl_allow_registration': {
+        'key': 'ytdl_allow_registration',
+        'path': 'YoutubeDLMaterial.Users.allow_registration'
+    },
+    'ytdl_auth_method': {
+        'key': 'ytdl_auth_method',
+        'path': 'YoutubeDLMaterial.Users.auth_method'
+    },
+    'ytdl_ldap_config': {
+        'key': 'ytdl_ldap_config',
+        'path': 'YoutubeDLMaterial.Users.ldap_config'
+    },
+
+    // Database
+    'ytdl_use_local_db': {
+        'key': 'ytdl_use_local_db',
+        'path': 'YoutubeDLMaterial.Database.use_local_db'
+    },
+    'ytdl_mongodb_connection_string': {
+        'key': 'ytdl_mongodb_connection_string',
+        'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
+    },
+
     // Advanced
+    'ytdl_default_downloader': {
+        'key': 'ytdl_default_downloader',
+        'path': 'YoutubeDLMaterial.Advanced.default_downloader'
+    },
     'ytdl_use_default_downloading_agent': {
         'key': 'ytdl_use_default_downloading_agent',
         'path': 'YoutubeDLMaterial.Advanced.use_default_downloading_agent'
@@ -91,10 +193,117 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_custom_downloading_agent',
         'path': 'YoutubeDLMaterial.Advanced.custom_downloading_agent'
     },
+    'ytdl_multi_user_mode': {
+        'key': 'ytdl_multi_user_mode',
+        'path': 'YoutubeDLMaterial.Advanced.multi_user_mode'
+    },
     'ytdl_allow_advanced_download': {
         'key': 'ytdl_allow_advanced_download',
         'path': 'YoutubeDLMaterial.Advanced.allow_advanced_download'
     },
+    'ytdl_use_cookies': {
+        'key': 'ytdl_use_cookies',
+        'path': 'YoutubeDLMaterial.Advanced.use_cookies'
+    },
+    'ytdl_jwt_expiration': {
+        'key': 'ytdl_jwt_expiration',
+        'path': 'YoutubeDLMaterial.Advanced.jwt_expiration'
+    },
+    'ytdl_logger_level': {
+        'key': 'ytdl_logger_level',
+        'path': 'YoutubeDLMaterial.Advanced.logger_level'
+    }
 };
 
-module.exports.CONFIG_ITEMS = CONFIG_ITEMS;
+exports.AVAILABLE_PERMISSIONS = [
+    'filemanager',
+    'settings',
+    'subscriptions',
+    'sharing',
+    'advanced_download',
+    'downloads_manager',
+    'tasks_manager'
+];
+
+exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
+
+// args that have a value after it (e.g. -o <output> or -f <format>)
+const YTDL_ARGS_WITH_VALUES = [
+    '--default-search',
+    '--config-location',
+    '--proxy',
+    '--socket-timeout',
+    '--source-address',
+    '--geo-verification-proxy',
+    '--geo-bypass-country',
+    '--geo-bypass-ip-block',
+    '--playlist-start',
+    '--playlist-end',
+    '--playlist-items',
+    '--match-title',
+    '--reject-title',
+    '--max-downloads',
+    '--min-filesize',
+    '--max-filesize',
+    '--date',
+    '--datebefore',
+    '--dateafter',
+    '--min-views',
+    '--max-views',
+    '--match-filter',
+    '--age-limit',
+    '--download-archive',
+    '-r',
+    '--limit-rate',
+    '-R',
+    '--retries',
+    '--fragment-retries',
+    '--buffer-size',
+    '--http-chunk-size',
+    '--external-downloader',
+    '--external-downloader-args',
+    '-a',
+    '--batch-file',
+    '-o',
+    '--output',
+    '--output-na-placeholder',
+    '--autonumber-start',
+    '--load-info-json',
+    '--cookies',
+    '--cache-dir',
+    '--encoding',
+    '--user-agent',
+    '--referer',
+    '--add-header',
+    '--sleep-interval',
+    '--max-sleep-interval',
+    '-f',
+    '--format',
+    '--merge-output-format',
+    '--sub-format',
+    '--sub-lang',
+    '-u',
+    '--username',
+    '-p',
+    '--password',
+    '-2',
+    '--twofactor',
+    '--video-password',
+    '--ap-mso',
+    '--ap-username',
+    '--ap-password',
+    '--audio-format',
+    '--audio-quality',
+    '--recode-video',
+    '--postprocessor-args',
+    '--metadata-from-title',
+    '--fixup',
+    '--ffmpeg-location',
+    '--exec',
+    '--convert-subs'
+];
+
+// we're using a Set here for performance
+exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
+
+exports.CURRENT_VERSION = 'v4.3';

backend/downloader.js

@@ -0,0 +1,640 @@
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+const path = require('path');
+const mergeFiles = require('merge-files');
+const NodeID3 = require('node-id3')
+const Mutex = require('async-mutex').Mutex;
+
+const youtubedl = require('youtube-dl');
+
+const logger = require('./logger');
+const config_api = require('./config');
+const twitch_api = require('./twitch');
+const { create } = require('xmlbuilder2');
+const categories_api = require('./categories');
+const utils = require('./utils');
+const db_api = require('./db');
+
+const mutex = new Mutex();
+let should_check_downloads = true;
+
+if (db_api.database_initialized) {
+    setupDownloads();
+} else {
+    db_api.database_initialized_bs.subscribe(init => {
+        if (init) setupDownloads();
+    });
+}
+
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
+    return await mutex.runExclusive(async () => {
+        const download = {
+            url: url,
+            type: type,
+            title: '',
+            user_uid: user_uid,
+            sub_id: sub_id,
+            sub_name: sub_name,
+            prefetched_info: prefetched_info,
+            options: options,
+            uid: uuid(),
+            step_index: 0,
+            paused: false,
+            running: false,
+            finished_step: true,
+            error: null,
+            percent_complete: null,
+            finished: false,
+            timestamp_start: Date.now()
+        };
+        await db_api.insertRecordIntoTable('download_queue', download);
+    
+        should_check_downloads = true;
+        return download;
+    });
+}
+
+exports.pauseDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        logger.warn(`Download ${download_uid} is already paused!`);
+        return false;
+    } else if (download['finished']) {
+        logger.info(`Download ${download_uid} could not be paused before completing.`);
+        return false;
+    }
+
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
+}
+
+exports.resumeDownload = async (download_uid) => {
+    return await mutex.runExclusive(async () => {
+        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+        if (!download['paused']) {
+            logger.warn(`Download ${download_uid} is not paused!`);
+            return false;
+        }
+
+        const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
+        should_check_downloads = true;
+        return success;
+    })
+}
+
+exports.restartDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    await exports.clearDownload(download_uid);
+    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    
+    should_check_downloads = true;
+    return success;
+}
+
+exports.cancelDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['cancelled']) {
+        logger.warn(`Download ${download_uid} is already cancelled!`);
+        return false;
+    } else if (download['finished']) {
+        logger.info(`Download ${download_uid} could not be cancelled before completing.`);
+        return false;
+    }
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
+}
+
+exports.clearDownload = async (download_uid) => {
+    return await db_api.removeRecord('download_queue', {uid: download_uid});
+}
+
+async function handleDownloadError(download_uid, error_message) {
+    if (!download_uid) return;
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+}
+
+async function setupDownloads() {
+    await fixDownloadState();
+    setInterval(checkDownloads, 1000);
+}
+
+async function fixDownloadState() {
+    const downloads = await db_api.getRecords('download_queue');
+    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
+    const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
+    for (let i = 0; i < running_downloads.length; i++) {
+        const running_download = running_downloads[i];
+        const update_obj = {finished_step: true, paused: true, running: false};
+        if (running_download['step_index'] > 0) {
+            update_obj['step_index'] = running_download['step_index'] - 1;
+        }
+        await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
+    }
+}
+
+async function checkDownloads() {
+    if (!should_check_downloads) return;
+
+    const downloads = await db_api.getRecords('download_queue');
+    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
+
+    await mutex.runExclusive(async () => {
+        // avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
+        const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
+        if (running_downloads.length === 0) {
+            should_check_downloads = false;
+            logger.verbose('Disabling checking downloads as none are available.');
+        }
+        return;
+    });
+
+    let running_downloads_count = downloads.filter(download => download['running']).length;
+    const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
+    for (let i = 0; i < waiting_downloads.length; i++) {
+        const waiting_download = waiting_downloads[i];
+        const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
+        if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
+
+        if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            // move to next step
+            running_downloads_count++;
+            if (waiting_download['step_index'] === 0) {
+                collectInfo(waiting_download['uid']);
+            } else if (waiting_download['step_index'] === 1) {
+                downloadQueuedFile(waiting_download['uid']);
+            }
+        }
+    }
+}
+
+async function collectInfo(download_uid) {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        return;
+    }
+    logger.verbose(`Collecting info for download ${download_uid}`);
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
+
+    const url = download['url'];
+    const type = download['type'];
+    const options = download['options'];
+
+    if (download['user_uid'] && !options.customFileFolderPath) {
+        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        const user_path = path.join(usersFileFolder, download['user_uid'], type);
+        options.customFileFolderPath = user_path + path.sep;
+    }
+
+    let args = await exports.generateArgs(url, type, options, download['user_uid']);
+
+    // get video info prior to download
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
+
+    if (!info) {
+        // info failed, error presumably already recorded
+        return;
+    }
+
+    let category = null;
+
+    // check if it fits into a category. If so, then get info again using new args
+    if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
+
+    // set custom output if the category has one and re-retrieve info so the download manager has the right file name
+    if (category && category['custom_output']) {
+        options.customOutput = category['custom_output'];
+        options.noRelativePath = true;
+        args = await exports.generateArgs(url, type, options, download['user_uid']);
+        info = await exports.getVideoInfoByURL(url, args, download_uid);
+    }
+
+    download['category'] = category;
+
+    // setup info required to calculate download progress
+
+    const expected_file_size = utils.getExpectedFileSize(info);
+
+    const files_to_check_for_progress = [];
+
+    // store info in download for future use
+    if (Array.isArray(info)) {
+        for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
+    } else {
+        files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
+    }
+
+    const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
+                                                                    finished_step: true,
+                                                                    running: false,
+                                                                    options: options,
+                                                                    files_to_check_for_progress: files_to_check_for_progress,
+                                                                    expected_file_size: expected_file_size,
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    prefetched_info: null
+                                                                });
+}
+
+async function downloadQueuedFile(download_uid) {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        return;
+    }
+    logger.verbose(`Downloading ${download_uid}`);
+    return new Promise(async resolve => {
+        const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+        const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+        await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
+
+        const url = download['url'];
+        const type = download['type'];
+        const options = download['options'];
+        const args = download['args'];
+        const category = download['category'];
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
+        if (options.customFileFolderPath) {
+            fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
+        }
+        fs.ensureDirSync(fileFolderPath);
+
+        const start_time = Date.now();
+
+        const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
+
+        // download file
+        youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
+            const file_objs = [];
+            let end_time = Date.now();
+            let difference = (end_time - start_time)/1000;
+            logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
+            clearInterval(download_checker);
+            if (err) {
+                logger.error(err.stderr);
+                await handleDownloadError(download_uid, err.stderr);
+                resolve(false);
+                return;
+            } else if (output) {
+                if (output.length === 0 || output[0].length === 0) {
+                    // ERROR!
+                    const error_message = `No output received for video download, check if it exists in your archive.`;
+                    await handleDownloadError(download_uid, error_message);
+                    logger.warn(error_message);
+                    resolve(false);
+                    return;
+                }
+
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+
+                    if (!output_json) {
+                        continue;
+                    }
+
+                    // get filepath with no extension
+                    const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
+
+                    const ext = type === 'audio' ? '.mp3' : '.mp4';
+                    var full_file_path = filepath_no_extension + ext;
+                    var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
+
+                    if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
+                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                            let vodId = url.split('twitch.tv/videos/')[1];
+                            vodId = vodId.split('?')[0];
+                            twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
+                    }
+
+                    // renames file if necessary due to bug
+                    if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
+                        try {
+                            fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
+                            logger.info('Renamed ' + file_name + '.webm to ' + file_name);
+                        } catch(e) {
+                            logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
+                        }
+                    }
+
+                    if (type === 'audio') {
+                        let tags = {
+                            title: output_json['title'],
+                            artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
+                        }
+                        let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
+                        if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
+                    }
+
+                    if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
+                        exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
+                    }
+
+                    if (options.cropFileSettings) {
+                        await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
+                    }
+
+                    // registers file in DB
+                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+
+                    file_objs.push(file_obj);
+                }
+
+                if (options.merged_string !== null && options.merged_string !== undefined) {
+                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
+                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
+                    const diff = current_merged_archive.replace(options.merged_string, '');
+                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+                    fs.appendFileSync(archive_path, diff);
+                }
+
+                let container = null;
+
+                if (file_objs.length > 1) {
+                    // create playlist
+                    const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                } else if (file_objs.length === 1) {
+                    container = file_objs[0];
+                } else {
+                    const error_message = 'Downloaded file failed to result in metadata object.';
+                    logger.error(error_message);
+                    await handleDownloadError(download_uid, error_message);
+                }
+
+                const file_uids = file_objs.map(file_obj => file_obj.uid);
+                await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
+                resolve();
+            }
+        });
+    });
+}
+
+// helper functions
+
+exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+
+    const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+    const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+
+    const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+    const globalArgs = config_api.getConfigItem('ytdl_custom_args');
+    const useCookies = config_api.getConfigItem('ytdl_use_cookies');
+    const is_audio = type === 'audio';
+
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
+
+    if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
+
+    const customArgs = options.customArgs;
+    let customOutput = options.customOutput;
+    const customQualityConfiguration = options.customQualityConfiguration;
+
+    // video-specific args
+    const selectedHeight = options.selectedHeight;
+
+    // audio-specific args
+    const maxBitrate = options.maxBitrate;
+
+    const youtubeUsername = options.youtubeUsername;
+    const youtubePassword = options.youtubePassword;
+
+    let downloadConfig = null;
+    let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
+    const is_youtube = url.includes('youtu');
+    if (!is_audio && !is_youtube) {
+        // tiktok videos fail when using the default format
+        qualityPath = null;
+    }
+
+    if (customArgs) {
+        downloadConfig = customArgs.split(',,');
+    } else {
+        if (customQualityConfiguration) {
+            qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
+        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
+            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}]`];
+        } else if (is_audio) {
+            qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
+        }
+
+        if (customOutput) {
+            customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
+            downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
+        } else {
+            downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
+        }
+
+        if (qualityPath) downloadConfig.push(...qualityPath);
+
+        if (is_audio && !options.skip_audio_args) {
+            downloadConfig.push('-x');
+            downloadConfig.push('--audio-format', 'mp3');
+        }
+
+        if (youtubeUsername && youtubePassword) {
+            downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
+        }
+
+        if (useCookies) {
+            if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+                downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
+            } else {
+                logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
+            }
+        }
+
+        const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
+        const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
+        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
+            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
+        }
+
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
+            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+
+            await fs.ensureDir(archive_folder);
+            await fs.ensureFile(archive_path);
+
+            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
+            await fs.ensureFile(blacklist_path);
+
+            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
+            await fs.ensureFile(merged_path);
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            await mergeFiles(inputPathList, merged_path);
+
+            options.merged_string = await fs.readFile(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
+        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+            downloadConfig.push('--write-thumbnail');
+        }
+
+        if (globalArgs && globalArgs !== '') {
+            // adds global args
+            if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
+                // if global args has an output, replce the original output with that of global args
+                const original_output_index = downloadConfig.indexOf('-o');
+                downloadConfig.splice(original_output_index, 2);
+            }
+            downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
+        }
+
+        if (options.additionalArgs && options.additionalArgs !== '') {
+            downloadConfig = utils.injectArgs(downloadConfig, options.additionalArgs.split(',,'));
+        }
+
+        const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
+        if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
+            downloadConfig.push('-r', rate_limit);
+        }
+        
+        if (default_downloader === 'yt-dlp') {
+            downloadConfig.push('--no-clean-infojson');
+        }
+
+    }
+
+    // filter out incompatible args
+    downloadConfig = filterArgs(downloadConfig, is_audio);
+
+    if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
+    return downloadConfig;
+}
+
+exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
+    return new Promise(resolve => {
+        // remove bad args
+        const new_args = [...args];
+
+        const archiveArgIndex = new_args.indexOf('--download-archive');
+        if (archiveArgIndex !== -1) {
+            new_args.splice(archiveArgIndex, 2);
+        }
+
+        new_args.push('--dump-json');
+
+        youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
+            if (output) {
+                let outputs = [];
+                try {
+                    for (let i = 0; i < output.length; i++) {
+                        let output_json = null;
+                        try {
+                            output_json = JSON.parse(output[i]);
+                        } catch(e) {
+                            output_json = null;
+                        }
+    
+                        if (!output_json) {
+                            continue;
+                        }
+
+                        outputs.push(output_json);
+                    }
+                    resolve(outputs.length === 1 ? outputs[0] : outputs);
+                } catch(e) {
+                    const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
+                    logger.error(error);
+                    if (download_uid) {
+                        await handleDownloadError(download_uid, error);
+                    }
+                    resolve(null);
+                }
+            } else {
+                let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
+                if (err.stderr) error_message += `\n\n${err.stderr}`;
+                logger.error(error_message);
+                if (download_uid) {
+                    await handleDownloadError(download_uid, error_message);
+                }
+                resolve(null);
+            }
+        });
+    });
+}
+
+function filterArgs(args, isAudio) {
+    const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
+    const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
+}
+
+async function checkDownloadPercent(download_uid) {
+    /*
+    This is more of an art than a science, we're just selecting files that start with the file name,
+    thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
+
+    Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
+    be divided by the "total expected bytes."
+    */
+
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    const files_to_check_for_progress = download['files_to_check_for_progress'];
+    const resulting_file_size = download['expected_file_size'];
+
+    if (!resulting_file_size) return;
+
+    let sum_size = 0;
+    for (let i = 0; i < files_to_check_for_progress.length; i++) {
+        const file_to_check_for_progress = files_to_check_for_progress[i];
+        const dir = path.dirname(file_to_check_for_progress);
+        if (!fs.existsSync(dir)) continue;
+        fs.readdir(dir, async (err, files) => {
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (!file.includes(path.basename(file_to_check_for_progress))) continue;
+                try {
+                    const file_stats = fs.statSync(path.join(dir, file));
+                    if (file_stats && file_stats.size) {
+                        sum_size += file_stats.size;
+                    }
+                } catch (e) {}
+            }
+            
+            const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+            await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
+        });
+    }
+}
+
+exports.generateNFOFile = (info, output_path) => {
+    const nfo_obj = {
+        episodedetails: {
+            title: info['fulltitle'],
+            episode: info['playlist_index'] ? info['playlist_index'] : undefined,
+            premiered: utils.formatDateString(info['upload_date']),
+            plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
+            director: info['artist'] ? info['artist'] : info['uploader']
+        }
+    };
+    const doc = create(nfo_obj);
+    const xml = doc.end({ prettyPrint: true });
+    fs.writeFileSync(output_path, xml);
+}
+
+function getArchiveFolder(fileFolderPath, options, user_uid) {
+    if (options.customArchivePath) {
+        return path.join(options.customArchivePath);
+    } else if (user_uid) {
+        return path.join(fileFolderPath, 'archives');
+    } else {
+        return path.join('appdata', 'archives');
+    }
+}

backend/ecosystem.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  apps : [{
+    name   : "YoutubeDL-Material",
+    script : "./app.js",
+    watch  : "placeholder",
+    watch_delay: 5000
+  }]
+}

backend/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -eu
+
+CMD="npm start"
+
+# if the first arg starts with "-" pass it to program
+if [ "${1#-}" != "$1" ]; then
+  set -- "$CMD" "$@"
+fi
+
+# chown current working directory to current user
+if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
+  find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
+  exec gosu "$UID:$GID" "$0" "$@"
+fi
+
+exec "$@"

backend/fix-scripts/001-fix_download_permissions.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-03
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines / Docker env defaults
+PATH_SUBS=/app/subscriptions
+PATH_AUDIO=/app/audio
+PATH_VIDS=/app/video
+
+clear -x
+echo "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+echo "Welcome to the INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M."
+echo "This script will set YTDL-M's download paths' owner to ${USER} (${UID}:${GID})"
+echo "and permissions to the default of 644."
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+echo "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBS ] && i=$((i+1)) && echo "✔ (${i}/3) Found Subscriptions directory at ${PATH_SUBS}"
+[ -d $PATH_AUDIO ] && i=$((i+1)) && echo "✔ (${i}/3) Found Audio directory at ${PATH_AUDIO}"
+[ -d $PATH_VIDS ] && i=$((i+1)) && echo "✔ (${i}/3) Found Video directory at ${PATH_VIDS}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    echo "\nCouldn't find any download path to fix permissions for! \nPlease edit this script to configure!"
+    exit 2;;
+  3)
+    echo "\nFound all download paths to fix permissions for. \nProceed? (Y/N)";;
+  *)
+    echo "\nOnly found ${i} out of 3 download paths! Something about this script's config must be wrong. \nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    echo "\n  Running jobs now... (this may take a while)\n"
+    [ -d $PATH_SUBS ] && chown "$UID:$GID" -R $PATH_SUBS && echo "✔ Set owner of ${PATH_SUBS} to ${USER}."
+    [ -d $PATH_SUBS ] && chmod 644 -R $PATH_SUBS && echo "✔ Set permissions of ${PATH_SUBS} to 644."
+    [ -d $PATH_AUDIO ] && chown "$UID:$GID" -R $PATH_AUDIO &&  echo "✔ Set owner of ${PATH_AUDIO} to ${USER}."
+    [ -d $PATH_AUDIO ] && chmod 644 -R $PATH_AUDIO && echo "✔ Set permissions of ${PATH_AUDIO} to 644."
+    [ -d $PATH_VIDS ] && chown "$UID:$GID" -R $PATH_VIDS &&  echo "✔ Set owner of ${PATH_VIDS} to ${USER}."
+    [ -d $PATH_VIDS ] && chmod 644 -R $PATH_VIDS && echo "✔ Set permissions of ${PATH_VIDS} to 644."
+    echo "\n✔ Done."
+    echo "\n  If you noticed file access errors those MAY be due to currently running downloads."
+    echo "  Feel free to re-run this script, however download parts should have correct file permissions anyhow. :)"
+    exit
+else
+    echo "\nOkay, bye."
+fi

backend/fix-scripts/002-fix_dupes_per_archive_file.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-09
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines (NO TRAILING SLASHES) / Docker env defaults
+PATH_SUBSARCHIVE=/app/subscriptions/archives
+PATH_ONEOFFARCHIVE=/app/appdata/archives
+
+# Backup paths (substitute with your personal preference if you like)
+PATH_SUBSARCHIVEBKP=$PATH_SUBSARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+PATH_ONEOFFARCHIVEBKP=$PATH_ONEOFFARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+
+
+# Define Colors for TUI
+yellow=$(tput setaf 3)
+normal=$(tput sgr0)
+
+tput civis                                                          # hide the cursor
+
+clear -x
+printf "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "Welcome to the INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M."
+printf "\nThis script will cycle through the archive files in the folders mentioned"
+printf "\nbelow and remove within each archive the dupe entries. (compact them)"
+printf "\nDuring some older builds of YTDL-M the archives could receive dupe"
+printf "\nentries and blow up in size, sometimes causing conflicts with download management."
+printf '\n%*s' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBSARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found Subscriptions archive directory at ${PATH_SUBSARCHIVE}"
+[ -d $PATH_ONEOFFARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found one-off archive directory at ${PATH_ONEOFFARCHIVE}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    printf "\n\n  Couldn't find any archive location path! \n\nPlease edit this script to configure!"
+    tput cnorm
+    exit 2;;
+  2)
+    printf "\n\n  Found all archive locations. \n\nProceed? (Y/N)";;
+  *)
+    printf "\n\n  Only found ${i} out of 2 archive locations! Something about this script's config must be wrong. \n\nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg          # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    printf "\n\nRunning jobs now... (this may take a while)\n"
+
+    printf "\nBacking up directories...\n"
+
+    chars="⣾⣽⣻⢿⡿⣟⣯⣷"
+    cp -R $PATH_SUBSARCHIVE $PATH_SUBSARCHIVEBKP &
+    PID=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.15
+    done
+    [ -d $PATH_SUBSARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_SUBSARCHIVE} to ${PATH_SUBSARCHIVEBKP} ($(du -sh $PATH_SUBSARCHIVEBKP | cut -f1))\n"
+
+    cp -R $PATH_ONEOFFARCHIVE $PATH_ONEOFFARCHIVEBKP &
+    PID2=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID2 ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.1
+    done
+    [ -d $PATH_ONEOFFARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_ONEOFFARCHIVE} to ${PATH_ONEOFFARCHIVEBKP} ($(du -sh $PATH_ONEOFFARCHIVEBKP | cut -f1))\n"
+
+
+    printf "\nCompacting files...\n"
+
+    tmpfile=$(mktemp) &&
+
+    [ -d $PATH_SUBSARCHIVE ] &&
+    find $PATH_SUBSARCHIVE -name '*.txt' -print0 | while read -d $'\0' file # Set delimiter to null because we want to catch all possible filenames (WE CANNOT CHANGE IFS HERE) - https://stackoverflow.com/a/15931055
+    do
+        cp "$file" "$tmpfile"
+        { awk '!x[$0]++' "$tmpfile" > "$file"; } &                         # https://unix.stackexchange.com/questions/159695/how-does-awk-a0-work
+        PID3=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID3 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [[ "$AFTER" -ne "$BEFORE" ]]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action needed for file: ${file}\n"
+        fi
+    done
+
+    [ -d $PATH_ONEOFFARCHIVE ] &&
+    find $PATH_ONEOFFARCHIVE -name '*.txt' -print0 | while read -d $'\0' file
+    do
+        cp "$file" "$tmpfile" &
+        awk '!x[$0]++' "$tmpfile" > "$file" &
+        PID4=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID4 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [ "$BEFORE" -ne "$AFTER" ]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action ran for file: ${file}\n"
+        fi
+    done
+    tput cnorm                                                     # show the cursor
+    rm "$tmpfile"
+
+    printf "\n\n✔ Done."
+    printf "\nℹ Please keep in mind that you may still want to"
+    printf "\n  run corruption checks against your archives!\n\n"
+    exit
+else
+    tput cnorm
+    printf "\nOkay, bye.\n\n"
+    exit
+fi

backend/logger.js

@@ -0,0 +1,23 @@
+const winston = require('winston');
+
+let debugMode = process.env.YTDL_MODE === 'debug';
+
+const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
+    return `${timestamp} ${level.toUpperCase()}: ${message}`;
+});
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(winston.format.timestamp(), defaultFormat),
+    defaultMeta: {},
+    transports: [
+      //
+      // - Write to all logs with level `info` and below to `combined.log`
+      // - Write all logs error (and below) to `error.log`.
+      //
+      new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
+      new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
+      new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
+    ]
+});
+
+module.exports = logger;

backend/package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "node app.js"
+    "start": "pm2-runtime --raw pm2.config.js",
+    "debug": "set YTDL_MODE=debug && node app.js"
   },
   "repository": {
     "type": "git",
@@ -18,13 +19,41 @@
   },
   "homepage": "",
   "dependencies": {
-    "archiver": "^3.1.1",
-    "async": "^3.1.0",
+    "archiver": "^5.3.1",
+    "async": "^3.2.3",
+    "async-mutex": "^0.3.1",
+    "axios": "^0.21.2",
+    "bcryptjs": "^2.4.0",
+    "compression": "^1.7.4",
     "config": "^3.2.3",
-    "exe": "^1.0.2",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
+    "fluent-ffmpeg": "^2.1.2",
+    "fs-extra": "^9.0.0",
+    "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
+    "md5": "^2.2.1",
+    "merge-files": "^0.1.2",
+    "mocha": "^9.2.2",
+    "moment": "^2.29.2",
+    "mongodb": "^3.6.9",
+    "multer": "1.4.5-lts.1",
+    "node-fetch": "^2.6.7",
+    "node-id3": "^0.1.14",
+    "node-schedule": "^2.1.0",
+    "passport": "^0.4.1",
+    "passport-http": "^0.3.0",
+    "passport-jwt": "^4.0.0",
+    "passport-ldapauth": "^3.0.1",
+    "passport-local": "^1.0.0",
+    "progress": "^2.0.3",
+    "ps-node": "^0.1.6",
+    "read-last-lines": "^1.7.2",
+    "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
+    "unzipper": "^0.10.10",
+    "uuidv4": "^6.0.6",
+    "winston": "^3.7.2",
+    "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"
   }
 }

backend/pm2.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+    apps : [{
+      name   : "YoutubeDL-Material",
+      script : "./app.js",
+      watch  : "placeholder",
+      out_file: "/dev/null",
+      error_file: "/dev/null"
+    }]
+}

backend/subscriptions.js

@@ -0,0 +1,538 @@
+const fs = require('fs-extra');
+const path = require('path');
+const youtubedl = require('youtube-dl');
+
+const config_api = require('./config');
+const utils = require('./utils');
+const logger = require('./logger');
+
+const debugMode = process.env.YTDL_MODE === 'debug';
+
+const db_api = require('./db');
+const downloader_api = require('./downloader');
+
+async function subscribe(sub, user_uid = null) {
+    const result_obj = {
+        success: false,
+        error: ''
+    };
+    return new Promise(async resolve => {
+        // sub should just have url and name. here we will get isPlaylist and path
+        sub.isPlaylist = sub.url.includes('playlist');
+        sub.videos = [];
+
+        let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
+
+        if (!sub.name && url_exists) {
+            logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
+            result_obj.error = 'Subcription with URL ' + sub.url + ' already exists! Custom name is required.';
+            resolve(result_obj);
+            return;
+        }
+
+        sub['user_uid'] = user_uid ? user_uid : undefined;
+        await db_api.insertRecordIntoTable('subscriptions', sub);
+
+        let success = await getSubscriptionInfo(sub);
+
+        if (success) {
+            getVideosForSub(sub, user_uid);
+        } else {
+            logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
+        }
+
+        result_obj.success = success;
+        result_obj.sub = sub;
+        resolve(result_obj);
+    });
+
+}
+
+async function getSubscriptionInfo(sub) {
+    // get videos
+    let downloadConfig = ['--dump-json', '--playlist-end', '1'];
+    let useCookies = config_api.getConfigItem('ytdl_use_cookies');
+    if (useCookies) {
+        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+            downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
+        } else {
+            logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
+        }
+    }
+
+    return new Promise(async resolve => {
+        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
+            if (debugMode) {
+                logger.info('Subscribe: got info for subscription ' + sub.id);
+            }
+            if (err) {
+                logger.error(err.stderr);
+                resolve(false);
+            } else if (output) {
+                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+                    logger.verbose('Could not get info for ' + sub.id);
+                    resolve(false);
+                }
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+                    if (!output_json) {
+                        continue;
+                    }
+                    if (!sub.name) {
+                        if (sub.isPlaylist) {
+                            sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
+                        } else {
+                            sub.name = output_json.uploader;
+                        }
+                        // if it's now valid, update
+                        if (sub.name) {
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
+                        }
+                    }
+
+                    // TODO: get even more info
+
+                    resolve(true);
+                }
+                resolve(false);
+            }
+        });
+    });
+}
+
+async function unsubscribe(sub, deleteMode, user_uid = null) {
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    let id = sub.id;
+
+    const sub_files = await db_api.getRecords('files', {sub_id: id});
+    for (let i = 0; i < sub_files.length; i++) {
+        const sub_file = sub_files[i];
+        if (config_api.descriptors[sub_file['uid']]) {
+            try {
+                for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
+                    config_api.descriptors[sub_file['uid']][i].destroy();
+                }
+            } catch(e) {
+                continue;
+            }
+        }
+    }
+
+    await db_api.removeRecord('subscriptions', {id: id});
+    await db_api.removeAllRecords('files', {sub_id: id});
+
+    // failed subs have no name, on unsubscribe they shouldn't error
+    if (!sub.name) {
+        return;
+    }
+
+    const appendedBasePath = getAppendedBasePath(sub, basePath);
+    if (deleteMode && (await fs.pathExists(appendedBasePath))) {
+        if (sub.archive && (await fs.pathExists(sub.archive))) {
+            const archive_file_path = path.join(sub.archive, 'archive.txt');
+            // deletes archive if it exists
+            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
+            if (await fs.pathExists(archive_file_path)) {
+                await fs.unlink(archive_file_path);
+            }
+            await fs.rmdir(sub.archive);
+        }
+        await fs.remove(appendedBasePath);
+    }
+}
+
+async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    // TODO: combine this with deletefile
+    let basePath = null;
+    basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
+                        : config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    const appendedBasePath = getAppendedBasePath(sub, basePath);
+    const name = file;
+    let retrievedID = null;
+
+    await db_api.removeRecord('files', {uid: file_uid});
+
+    let filePath = appendedBasePath;
+    const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
+    var jsonPath = path.join(__dirname,filePath,name+'.info.json');
+    var videoFilePath = path.join(__dirname,filePath,name+ext);
+    var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
+    var altImageFilePath = path.join(__dirname,filePath,name+'.webp');
+
+    const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
+        fs.pathExists(jsonPath),
+        fs.pathExists(videoFilePath),
+        fs.pathExists(imageFilePath),
+        fs.pathExists(altImageFilePath),
+    ]);
+
+    if (jsonExists) {
+        retrievedID = fs.readJSONSync(jsonPath)['id'];
+        await fs.unlink(jsonPath);
+    }
+
+    if (imageFileExists) {
+        await fs.unlink(imageFilePath);
+    }
+
+    if (altImageFileExists) {
+        await fs.unlink(altImageFilePath);
+    }
+
+    if (videoFileExists) {
+        await fs.unlink(videoFilePath);
+        if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
+            return false;
+        } else {
+            // check if the user wants the video to be redownloaded (deleteForever === false)
+            if (useArchive && retrievedID) {
+                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
+
+                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
+            }
+            return true;
+        }
+    } else {
+        // TODO: tell user that the file didn't exist
+        return true;
+    }
+}
+
+async function getVideosForSub(sub, user_uid = null) {
+    const latest_sub_obj = await getSubscription(sub.id);
+    if (!latest_sub_obj || latest_sub_obj['downloading']) {
+        return false;
+    }
+
+    updateSubscriptionProperty(sub, {downloading: true}, user_uid);
+
+    // get basePath
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    let appendedBasePath = getAppendedBasePath(sub, basePath);
+    fs.ensureDirSync(appendedBasePath);
+
+    const downloadConfig = await generateArgsForSubscription(sub, user_uid);
+
+    // get videos
+    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+
+    return new Promise(async resolve => {
+        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
+            // cleanup
+            updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+
+            logger.verbose('Subscription: finished check for ' + sub.name);
+            if (err && !output) {
+                logger.error(err.stderr ? err.stderr : err.message);
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
+                    logger.info('An error was encountered with at least one video, backup method will be used.')
+                    try {
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
+                        resolve(files_to_download);
+                    } catch(e) {
+                        logger.error('Backup method failed. See error below:');
+                        logger.error(e);
+                    }
+                } else {
+                    logger.error('Subscription check failed!');
+                }
+                resolve(false);
+            } else if (output) {
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
+                resolve(files_to_download);
+           }
+        });
+    }, err => {
+        logger.error(err);
+        updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+    });
+}
+
+async function handleOutputJSON(output, sub, user_uid) {
+    if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+        await setFreshUploads(sub, user_uid);
+        checkVideosForFreshUploads(sub, user_uid);
+    }
+
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+        logger.verbose('No additional videos to download for ' + sub.name);
+        return [];
+    }
+
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
+    const files_to_download = await getFilesToDownload(sub, output_jsons);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+    for (let j = 0; j < files_to_download.length; j++) {
+        const file_to_download = files_to_download[j];
+        file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
+    }
+
+    return files_to_download;
+}
+
+function generateOptionsForSubscriptionDownload(sub, user_uid) {
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+
+    const base_download_options = {
+        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        customFileFolderPath: getAppendedBasePath(sub, basePath),
+        customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
+        customArchivePath: path.join(basePath, 'archives', sub.name),
+        additionalArgs: sub.custom_args
+    }
+
+    return base_download_options;
+}
+
+async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
+    // get basePath
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+
+    let appendedBasePath = getAppendedBasePath(sub, basePath);
+
+    const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+
+    let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
+    if (desired_path) {
+        fullOutput = `"${desired_path}.%(ext)s"`;
+    } else if (sub.custom_output) {
+        fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
+    }
+
+    let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
+
+    let qualityPath = null;
+    if (sub.type && sub.type === 'audio') {
+        qualityPath = ['-f', 'bestaudio']
+        qualityPath.push('-x');
+        qualityPath.push('--audio-format', 'mp3');
+    } else {
+        if (!sub.maxQuality || sub.maxQuality === 'best') qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'];
+        else qualityPath = ['-f', `bestvideo[height<=${sub.maxQuality}]+bestaudio/best[height<=${sub.maxQuality}]`, '--merge-output-format', 'mp4'];
+    }
+
+    downloadConfig.push(...qualityPath)
+
+    if (sub.custom_args) {
+        const customArgsArray = sub.custom_args.split(',,');
+        if (customArgsArray.indexOf('-f') !== -1) {
+            // if custom args has a custom quality, replce the original quality with that of custom args
+            const original_output_index = downloadConfig.indexOf('-f');
+            downloadConfig.splice(original_output_index, 2);
+        }
+        downloadConfig.push(...customArgsArray);
+    }
+
+    let archive_dir = null;
+    let archive_path = null;
+
+    if (useArchive && !redownload) {
+        if (sub.archive) {
+            archive_dir = sub.archive;
+            if (sub.type && sub.type === 'audio') {
+                archive_path = path.join(archive_dir, 'merged_audio.txt');
+            } else {
+                archive_path = path.join(archive_dir, 'merged_video.txt');
+            }            
+        }
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
+    if (sub.timerange && !redownload) {
+        downloadConfig.push('--dateafter', sub.timerange);
+    }
+
+    let useCookies = config_api.getConfigItem('ytdl_use_cookies');
+    if (useCookies) {
+        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+            downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
+        } else {
+            logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
+        }
+    }
+
+    if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+        downloadConfig.push('--write-thumbnail');
+    }
+
+    const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
+    if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
+        downloadConfig.push('-r', rate_limit);
+    }
+
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+    if (default_downloader === 'yt-dlp') {
+        downloadConfig.push('--no-clean-infojson');
+    }
+
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
+    return downloadConfig;
+}
+
+async function getFilesToDownload(sub, output_jsons) {
+    const files_to_download = [];
+    for (let i = 0; i < output_jsons.length; i++) {
+        const output_json = output_jsons[i];
+        const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
+        if (file_missing) {
+            const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
+            if (file_with_path_exists) {
+                // or maybe just overwrite???
+                logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+            }
+            files_to_download.push(output_json);
+        }
+    }
+    return files_to_download;
+}
+
+
+async function getSubscriptions(user_uid = null) {
+    return await db_api.getRecords('subscriptions', {user_uid: user_uid});
+}
+
+async function getAllSubscriptions() {
+    const all_subs = await db_api.getRecords('subscriptions');
+    const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
+    return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
+}
+
+async function getSubscription(subID) {
+    return await db_api.getRecord('subscriptions', {id: subID});
+}
+
+async function getSubscriptionByName(subName, user_uid = null) {
+    return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
+}
+
+async function updateSubscription(sub) {
+    await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
+    return true;
+}
+
+async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
+    subs.forEach(async sub => {
+        await updateSubscriptionProperty(sub, assignment_obj);
+    });
+}
+
+async function updateSubscriptionProperty(sub, assignment_obj) {
+    // TODO: combine with updateSubscription
+    await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
+    return true;
+}
+
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    if (!sub_files) return;
+    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
+    sub_files.forEach(async file => {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
+            // set upload as fresh
+            const file_uid = file['uid'];
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
+        }
+    });
+}
+
+async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
+    sub_files.forEach(async file => {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
+            await checkVideoIfBetterExists(file, sub, user_uid)
+        }
+    });
+}
+
+async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
+    const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
+    const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
+    logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
+    // simulate a download to verify that a better version exists
+    youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
+        if (err) {
+            // video is not available anymore for whatever reason
+        } else if (output) {
+            const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
+            if (output[metric_to_compare] > file_obj[metric_to_compare]) {
+                // download new video as the simulated one is better
+                youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
+                    if (err) {
+                        logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
+                    } else if (output) {
+                        logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
+                    }
+                });
+            } 
+        }
+    });
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
+}
+
+// helper functions
+
+function getAppendedBasePath(sub, base_path) {
+    return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
+}
+
+module.exports = {
+    getSubscription        : getSubscription,
+    getSubscriptionByName  : getSubscriptionByName,
+    getSubscriptions       : getSubscriptions,
+    getAllSubscriptions    : getAllSubscriptions,
+    updateSubscription     : updateSubscription,
+    subscribe              : subscribe,
+    unsubscribe            : unsubscribe,
+    deleteSubscriptionFile : deleteSubscriptionFile,
+    getVideosForSub        : getVideosForSub,
+    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
+    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
+}

backend/subscriptions/channels/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

backend/subscriptions/playlists/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

backend/tasks.js

@@ -0,0 +1,196 @@
+const db_api = require('./db');
+const youtubedl_api = require('./youtube-dl');
+
+const fs = require('fs-extra');
+const logger = require('./logger');
+const scheduler = require('node-schedule');
+
+const TASKS = {
+    backup_local_db: {
+        run: db_api.backupDB,
+        title: 'Backup DB',
+        job: null
+    },
+    missing_files_check: {
+        run: checkForMissingFiles,
+        confirm: deleteMissingFiles,
+        title: 'Missing files check',
+        job: null
+    },
+    missing_db_records: {
+        run: db_api.importUnregisteredFiles,
+        title: 'Import missing DB records',
+        job: null
+    },
+    duplicate_files_check: {
+        run: checkForDuplicateFiles,
+        confirm: removeDuplicates,
+        title: 'Find duplicate files in DB',
+        job: null
+    },
+    youtubedl_update_check: {
+        run: youtubedl_api.checkForYoutubeDLUpdate,
+        confirm: youtubedl_api.updateYoutubeDL,
+        title: 'Update youtube-dl',
+        job: null
+    }
+}
+
+function scheduleJob(task_key, schedule) {
+    // schedule has to be converted from our format to one node-schedule can consume
+    let converted_schedule = null;
+    if (schedule['type'] === 'timestamp') {
+        converted_schedule = new Date(schedule['data']['timestamp']);
+    } else if (schedule['type'] === 'recurring') {
+        const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
+        const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
+        const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
+        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
+    } else {
+        logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
+        return null;
+    }
+
+    return scheduler.scheduleJob(converted_schedule, async () => {
+        const task_state = await db_api.getRecord('tasks', {key: task_key});
+        if (task_state['running'] || task_state['confirming']) {
+            logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
+            return;
+        }
+        
+        // remove schedule if it's a one-time task
+        if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
+        // we're just "running" the task, any confirmation should be user-initiated
+        exports.executeRun(task_key);
+    });
+}
+
+if (db_api.database_initialized) {
+    exports.setupTasks();
+} else {
+    db_api.database_initialized_bs.subscribe(init => {
+        if (init) exports.setupTasks();
+    });
+}
+
+exports.setupTasks = async () => {
+    const tasks_keys = Object.keys(TASKS);
+    for (let i = 0; i < tasks_keys.length; i++) {
+        const task_key = tasks_keys[i];
+        const task_in_db = await db_api.getRecord('tasks', {key: task_key});
+        if (!task_in_db) {
+            // insert task metadata into table if missing
+            await db_api.insertRecordIntoTable('tasks', {
+                key: task_key,
+                title: TASKS[task_key]['title'],
+                last_ran: null,
+                last_confirmed: null,
+                running: false,
+                confirming: false,
+                data: null,
+                error: null,
+                schedule: null,
+                options: {}
+            });
+        } else {
+            // reset task if necessary
+            await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
+
+            // schedule task and save job
+            if (task_in_db['schedule']) {
+                // prevent timestamp schedules from being set to the past
+                if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) {
+                    await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
+                    continue;
+                }
+                TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
+            }
+        }
+    }
+}
+
+exports.executeTask = async (task_key) => {
+    if (!TASKS[task_key]) {
+        logger.error(`Task ${task_key} does not exist!`);
+        return;
+    }
+    logger.verbose(`Executing task ${task_key}`);
+    await exports.executeRun(task_key);
+    if (!TASKS[task_key]['confirm']) return;
+    await exports.executeConfirm(task_key);
+    logger.verbose(`Finished executing ${task_key}`);
+}
+
+exports.executeRun = async (task_key) => {
+    logger.verbose(`Running task ${task_key}`);
+    // don't set running to true when backup up DB as it will be stick "running" if restored
+    if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
+    const data = await TASKS[task_key].run();
+    await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
+    logger.verbose(`Finished running task ${task_key}`);
+}
+
+exports.executeConfirm = async (task_key) => {
+    logger.verbose(`Confirming task ${task_key}`);
+    if (!TASKS[task_key]['confirm']) {
+        return null;
+    }
+    await db_api.updateRecord('tasks', {key: task_key}, {confirming: true});
+    const task_obj = await db_api.getRecord('tasks', {key: task_key});
+    const data = task_obj['data'];
+    await TASKS[task_key].confirm(data);
+    await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
+    logger.verbose(`Finished confirming task ${task_key}`);
+}
+
+exports.updateTaskSchedule = async (task_key, schedule) => {
+    logger.verbose(`Updating schedule for task ${task_key}`);
+    await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
+    if (TASKS[task_key]['job']) {
+        TASKS[task_key]['job'].cancel();
+        TASKS[task_key]['job'] = null;
+    }
+    if (schedule) {
+        TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
+    }
+}
+
+// missing files check
+
+async function checkForMissingFiles() {
+    const missing_files = [];
+    const all_files = await db_api.getRecords('files');
+    for (let i = 0; i < all_files.length; i++) {
+        const file_to_check = all_files[i];
+        const file_exists = fs.existsSync(file_to_check['path']);
+        if (!file_exists) missing_files.push(file_to_check['uid']);
+    }
+    return {uids: missing_files};
+}
+
+async function deleteMissingFiles(data) {
+    const uids = data['uids'];
+    for (let i = 0; i < uids.length; i++) {
+        const uid = uids[i];
+        await db_api.removeRecord('files', {uid: uid});
+    }
+}
+
+// duplicate files check
+
+async function checkForDuplicateFiles() {
+    const duplicate_files = await db_api.findDuplicatesByKey('files', 'path');
+    const duplicate_uids = duplicate_files.map(duplicate_file => duplicate_file['uid']);
+    if (duplicate_uids && duplicate_uids.length > 0) {
+        return {uids: duplicate_uids};
+    }
+    return {uids: []};
+}
+
+async function removeDuplicates(data) {
+    for (let i = 0; i < data['uids'].length; i++) {
+        await db_api.removeRecord('files', {uid: data['uids'][i]});
+    }
+}
+
+exports.TASKS = TASKS;

backend/test/tests.js

@@ -0,0 +1,629 @@
+const assert = require('assert');
+const low = require('lowdb')
+const winston = require('winston');
+const path = require('path');
+
+process.chdir('./backend')
+
+const FileSync = require('lowdb/adapters/FileSync');
+
+const adapter = new FileSync('./appdata/db.json');
+const db = low(adapter)
+
+const users_adapter = new FileSync('./appdata/users.json');
+const users_db = low(users_adapter);
+
+const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
+    return `${timestamp} ${level.toUpperCase()}: ${message}`;
+});
+
+let debugMode = process.env.YTDL_MODE === 'debug';
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(winston.format.timestamp(), defaultFormat),
+    defaultMeta: {},
+    transports: [
+      //
+      // - Write to all logs with level `info` and below to `combined.log`
+      // - Write all logs error (and below) to `error.log`.
+      //
+      new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
+      new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
+      new winston.transports.Console({level: 'debug', name: 'console'})
+    ]
+});
+
+var auth_api = require('../authentication/auth');
+var db_api = require('../db');
+const utils = require('../utils');
+const subscriptions_api = require('../subscriptions');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
+
+db_api.initialize(db, users_db);
+
+const sample_video_json = {
+    id: "Sample Video",
+    title: "Sample Video",
+    thumbnailURL: "https://sampleurl.jpg",
+    isAudio: false,
+    duration: 177.413,
+    url: "sampleurl.com",
+    uploader: "Sample Uploader",
+    size: 2838445,
+    path: "users\\admin\\video\\Sample Video.mp4",
+    upload_date: "2017-07-28",
+    description: null,
+    view_count: 230,
+    abr: 128,
+    thumbnailPath: null,
+    user_uid: "admin",
+    uid: "1ada04ab-2773-4dd4-bbdd-3e2d40761c50",
+    registered: 1628469039377
+}
+
+describe('Database', async function() {
+    describe('Import', async function() {
+        it('Migrate', async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords();
+            const success = await db_api.importJSONToDB(db.value(), users_db.value());
+            assert(success);
+        });
+
+        it('Transfer to remote', async function() {
+            await db_api.removeAllRecords('test');
+            await db_api.insertRecordIntoTable('test', {test: 'test'});
+
+            await db_api.transferDB(true);
+            const success = await db_api.getRecord('test', {test: 'test'});
+            assert(success);
+        });
+
+        it('Transfer to local', async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords('test');
+            await db_api.insertRecordIntoTable('test', {test: 'test'});
+
+            await db_api.transferDB(false);
+            const success = await db_api.getRecord('test', {test: 'test'});
+            assert(success);
+        });
+
+        it('Restore db', async function() {
+            const db_stats = await db_api.getDBStats();
+            
+            const file_name = await db_api.backupDB();
+            await db_api.restoreDB(file_name);
+
+            const new_db_stats = await db_api.getDBStats();
+
+            assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats));
+        });
+    });
+
+    describe('Export', function() {
+
+    });
+
+
+    describe('Basic functions', async function() {
+        beforeEach(async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords('test');
+        });
+        it('Add and read record', async function() {
+            this.timeout(120000);
+            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+            assert(added_record['test_add'] === 'test');
+            await db_api.removeRecord('test', {test_add: 'test'});
+        });
+
+        it('Find duplicates by key', async function() {
+            const test_duplicates = [
+                {
+                    test: 'testing',
+                    key: '1'
+                },
+                {
+                    test: 'testing',
+                    key: '2'
+                },
+                {
+                    test: 'testing_missing',
+                    key: '3'
+                },
+                {
+                    test: 'testing',
+                    key: '4'
+                }
+            ];
+            await db_api.insertRecordsIntoTable('test', test_duplicates);
+            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
+            console.log(duplicates);
+        });
+
+        it('Update record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+            assert(updated_record['added_field']);
+            await db_api.removeRecord('test', {test_update: 'test'});
+        });
+
+        it('Remove record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+            assert(delete_succeeded);
+            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+            assert(!deleted_record);
+        });
+
+        it('Push to record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 1);
+        });
+
+        it('Pull from record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 0);
+        });
+
+        it('Bulk add', async function() {
+            this.timeout(120000);
+            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+            const test_records = [];
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                test_records.push({
+                    uid: uuid()
+                });
+            }
+            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+
+            const received_records = await db_api.getRecords('test');
+            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+        });
+
+        it('Bulk update', async function() {
+            // bulk add records
+            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+            const test_records = [];
+            const update_obj = {};
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                const test_uid =  uuid();
+                test_records.push({
+                    uid: test_uid
+                });
+                update_obj[test_uid] = {added_field: true};
+            }
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            assert(success);
+
+            // makes sure they are added
+            const received_records = await db_api.getRecords('test');
+            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
+            assert(success);
+
+            const received_updated_records = await db_api.getRecords('test');
+            for (let i = 0; i < received_updated_records.length; i++) {
+                success &= received_updated_records[i]['added_field'];
+            }
+            assert(success);
+        });
+
+        it('Stats', async function() {
+            const stats = await db_api.getDBStats();
+            assert(stats);
+        });
+
+        it('Query speed', async function() {
+            this.timeout(120000); 
+            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+            const test_records = [];
+            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                const uid = uuid();
+                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+            }
+            const insert_start = Date.now();
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            const insert_end = Date.now();
+
+            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+            const query_start = Date.now();
+            const random_record = await db_api.getRecord('test', {uid: random_uid});
+            const query_end = Date.now();
+
+            console.log(random_record)
+
+            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+            success = !!random_record;
+
+            assert(success);
+        });
+    });
+
+    describe('Local DB Filters', async function() {
+        it('Basic', async function() {
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: 'test'}, 'find');
+            assert(result && result['test'] === 'test');
+        });
+
+        it('Regex', async function() {
+            const filter = {$regex: `\\w+\\d`, $options: 'i'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Not equals', async function() {
+            const filter = {$ne: 'test'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Nested', async function() {
+            const result = db_api.applyFilterLocalDB([{test1: {test2: 'test3'}}, {test4: 'test5'}], {'test1.test2': 'test3'}, 'find');
+            assert(result && result['test1']['test2'] === 'test3');
+        });
+    })
+});
+
+describe('Multi User', async function() {
+    let user = null;
+    const user_to_test = 'admin';
+    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
+    const playlist_to_test = 'ysabVZz4x';
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        auth_api.initialize(db_api, logger);
+        subscriptions_api.initialize(db_api, logger);
+        user = await auth_api.login('admin', 'pass');
+    });
+    describe('Authentication', function() {
+        it('login', async function() {
+            assert(user);
+        });
+    });
+    describe('Video player - normal', async function() {
+        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+        await db_api.insertRecordIntoTable('files', sample_video_json);
+        const video_to_test = sample_video_json['uid'];
+        it('Get video', async function() {
+            const video_obj = await db_api.getVideo(video_to_test);
+            assert(video_obj);
+        });
+
+        it('Video access - disallowed', async function() {
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            assert(!video_obj);
+        });
+
+        it('Video access - allowed', async function() {
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            assert(video_obj);
+        });
+    });
+    describe('Zip generators', function() {
+        it('Playlist zip generator', async function() {
+            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
+            assert(playlist);
+            const playlist_files_to_download = [];
+            for (let i = 0; i < playlist['uids'].length; i++) {
+                const uid = playlist['uids'][i];
+                const playlist_file = await db_api.getVideo(uid, user_to_test);
+                playlist_files_to_download.push(playlist_file);
+            }
+            const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+
+        it('Subscription zip generator', async function() {
+            const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
+            const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
+            assert(sub);
+            const sub_files_to_download = [];
+            for (let i = 0; i < sub_videos.length; i++) {
+                const sub_file = sub_videos[i];
+                sub_files_to_download.push(sub_file);
+            }
+            const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+    });
+    // describe('Video player - subscription', function() {
+    //     const sub_to_test = '';
+    //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
+    //     it('Get video', async function() {
+    //         const video_obj = db_api.getVideo(video_to_test, 'admin', );
+    //         assert(video_obj);
+    //     });
+
+    //     it('Video access - disallowed', async function() {
+    //         await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test);
+    //         const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+    //         assert(!video_obj);
+    //     });
+
+    //     it('Video access - allowed', async function() {
+    //         await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test);
+    //         const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+    //         assert(video_obj);
+    //     });
+    // });
+
+});
+    
+describe('Downloader', function() {
+    const downloader_api = require('../downloader');
+    const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
+    const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
+    const options = {
+        ui_uid: uuid(),
+        user: 'admin'
+    }
+
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('download_queue');
+    });
+
+    it('Get file info', async function() {
+        this.timeout(300000);
+        const info = await downloader_api.getVideoInfoByURL(url);
+        assert(!!info);
+    });
+
+    it('Download file', async function() {
+        this.timeout(300000); 
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        console.log(returned_download);
+        await utils.wait(20000);
+
+    });
+
+    it('Tag file', async function() {
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        const tags = {
+            title: sample_json['title'],
+            artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
+            TRCK: '27'
+        }
+        NodeID3.write(tags, audio_path);
+        const written_tags = NodeID3.read(audio_path);
+        assert(written_tags['raw']['TRCK'] === '27');
+    });
+
+    it('Queue file', async function() {
+        this.timeout(300000); 
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        console.log(returned_download);
+        await utils.wait(20000);
+    });
+
+    it('Pause file', async function() {
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        await downloader_api.pauseDownload(returned_download['uid']);
+        const updated_download = await db_api.getRecord('download_queue', {uid: returned_download['uid']});
+        assert(updated_download['paused'] && !updated_download['running']);
+    });
+
+    it('Generate args', async function() {
+        const args = await downloader_api.generateArgs(url, 'video', options);
+        assert(args.length > 0);
+    });
+
+    it('Generate args - subscription', async function() {
+        const sub = await subscriptions_api.getSubscription(sub_id);
+        const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
+        const args_normal = await downloader_api.generateArgs(url, 'video', options);
+        const args_sub = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        console.log(JSON.stringify(args_normal) !== JSON.stringify(args_sub));
+    });
+
+    it('Generate kodi NFO file', async function() {
+        const nfo_file_path = './test/sample.nfo';
+        if (fs.existsSync(nfo_file_path)) {
+            fs.unlinkSync(nfo_file_path);
+        }
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        downloader_api.generateNFOFile(sample_json, nfo_file_path);
+        assert(fs.existsSync(nfo_file_path), true);
+        fs.unlinkSync(nfo_file_path);
+    });
+
+    it('Inject args', async function() {
+        const original_args1 = ['--no-resize-buffer', '-o', '%(title)s', '--no-mtime'];
+        const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        const updated_args1 = utils.injectArgs(original_args1, new_args1);
+        const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
+
+        const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
+        const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        const updated_args2 = utils.injectArgs(original_args2, new_args2);
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
+        console.log(updated_args2);
+        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
+    });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1493770675';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
+    });
+});
+
+describe('Tasks', function() {
+    const tasks_api = require('../tasks');
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('tasks');
+
+        const dummy_task = {
+            run: async () => { await utils.wait(500); return true; },
+            confirm: async () => { await utils.wait(500); return true; },
+            title: 'Dummy task',
+            job: null
+        };
+        tasks_api.TASKS['dummy_task'] = dummy_task;
+
+        await tasks_api.setupTasks();
+    });
+    it('Backup db', async function() {
+        const backups_original = await utils.recFindByExt('appdata', 'bak');
+        const original_length = backups_original.length;
+        await tasks_api.executeTask('backup_local_db');
+        const backups_new = await utils.recFindByExt('appdata', 'bak');
+        const new_length = backups_new.length;
+        assert(original_length, new_length-1);
+    });
+
+    it('Check for missing files', async function() {
+        this.timeout(300000);
+        await db_api.removeAllRecords('files', {uid: 'test'});
+        const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
+        await db_api.insertRecordIntoTable('files', test_missing_file);
+        await tasks_api.executeTask('missing_files_check');
+        const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
+        assert(!missing_file_db_record, true);
+    });
+
+    it('Check for duplicate files', async function() {
+        this.timeout(300000);
+        await db_api.removeAllRecords('files', {uid: 'test1'});
+        await db_api.removeAllRecords('files', {uid: 'test2'});
+        const test_duplicate_file1 = {uid: 'test1', path: 'test/missing_file.mp4'};
+        const test_duplicate_file2 = {uid: 'test2', path: 'test/missing_file.mp4'};
+        const test_duplicate_file3 = {uid: 'test3', path: 'test/missing_file.mp4'};
+        await db_api.insertRecordIntoTable('files', test_duplicate_file1);
+        await db_api.insertRecordIntoTable('files', test_duplicate_file2);
+        await db_api.insertRecordIntoTable('files', test_duplicate_file3);
+
+        await tasks_api.executeRun('duplicate_files_check');
+        const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
+        assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
+
+        await tasks_api.executeTask('duplicate_files_check');
+        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
+        assert(duplicated_record_count == 1, true);
+    });
+
+    it('Import unregistered files', async function() {
+        this.timeout(300000);
+
+        // pre-test cleanup
+        await db_api.removeAllRecords('files', {title: 'Sample File'});
+        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
+        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
+
+        // copies in files
+        fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
+        fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
+        await tasks_api.executeTask('missing_db_records');
+        const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
+        assert(!!imported_file, true);
+        
+        // post-test cleanup
+        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
+        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
+    });
+
+    it('Schedule and cancel task', async function() {
+        this.timeout(5000);
+        const today_one_year = new Date();
+        today_one_year.setFullYear(today_one_year.getFullYear() + 1);
+        const schedule_obj = {
+            type: 'timestamp',
+            data: { timestamp: today_one_year.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        const dummy_task = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
+        assert(!!dummy_task['schedule']);
+
+        await tasks_api.updateTaskSchedule('dummy_task', null);
+        const dummy_task_updated = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!tasks_api.TASKS['dummy_task']['job']);
+        assert(!dummy_task_updated['schedule']);
+    });
+
+    it('Schedule and run task', async function() {
+        this.timeout(5000);
+        const today_1_second = new Date();
+        today_1_second.setSeconds(today_1_second.getSeconds() + 1);
+        const schedule_obj = {
+            type: 'timestamp',
+            data: { timestamp: today_1_second.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
+        await utils.wait(2000);
+        const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(dummy_task_obj['data']);
+    });
+});
+
+describe('Archive', async function() {
+    const archive_path = path.join('test', 'archives');
+    fs.ensureDirSync(archive_path);
+    const archive_file_path = path.join(archive_path, 'archive_video.txt');
+    const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
+    beforeEach(async function() {
+        if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
+        fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
+
+        if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
+        fs.writeFileSync(blacklist_file_path, '');
+    });
+    
+    it('Delete from archive', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
+        const new_archive = fs.readFileSync(archive_file_path);
+        assert(!new_archive.includes('testing2'));
+    });
+
+    it('Delete from archive - blacklist', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
+        const new_archive = fs.readFileSync(archive_file_path);
+        const new_blacklist = fs.readFileSync(blacklist_file_path);
+        assert(!new_archive.includes('testing2'));
+        assert(new_blacklist.includes('testing2'));
+    });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
+    });
+});

backend/twitch.js

@@ -0,0 +1,115 @@
+const config_api = require('./config');
+const logger = require('./logger');
+
+const moment = require('moment');
+const fs = require('fs-extra')
+const path = require('path');
+
+async function getCommentsForVOD(clientID, clientSecret, vodId) {
+    const { promisify } = require('util');
+    const child_process = require('child_process');
+    const exec = promisify(child_process.exec);
+    
+    // Reject invalid params to prevent command injection attack
+    if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
+        logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
+        return null;
+    }
+
+    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
+
+    if (result['stderr']) {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
+        logger.error(result['stderr']);
+        return null;
+    }
+
+    const temp_chat_path = path.join('appdata', `${vodId}.json`);
+
+    const raw_json = fs.readJSONSync(temp_chat_path);
+    const new_json = raw_json.comments.map(comment_obj => {
+        return {
+            timestamp: comment_obj.content_offset_seconds,
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+            name: comment_obj.commenter.name,
+            message: comment_obj.message.body,
+            user_color: comment_obj.message.user_color
+        }
+    });
+
+    fs.unlinkSync(temp_chat_path);
+
+    return new_json;
+}
+
+async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    let file_path = null;
+
+    if (user_uid) {
+        if (sub) {
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
+        } else {
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
+        }
+    } else {
+        if (sub) {
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
+        } else {
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
+        }
+    }
+
+    var chat_file = null;
+    if (fs.existsSync(file_path)) {
+        chat_file = fs.readJSONSync(file_path);
+    }
+
+    return chat_file;
+}
+
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
+
+    // save file if needed params are included
+    let file_path = null;
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
+        if (sub) {
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
+        } else {
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
+        }
+    } else {
+        if (sub) {
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
+        } else {
+            file_path = path.join(type, `${id}.twitch_chat.json`);
+        }
+    }
+
+    if (chat) fs.writeJSONSync(file_path, chat);
+
+    return chat;
+}
+
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
+module.exports = {
+    getCommentsForVOD: getCommentsForVOD,
+    getTwitchChatByFileID: getTwitchChatByFileID,
+    downloadTwitchChatByVODID: downloadTwitchChatByVODID
+}

backend/utils.js

@@ -0,0 +1,591 @@
+const fs = require('fs-extra');
+const path = require('path');
+const ffmpeg = require('fluent-ffmpeg');
+const archiver = require('archiver');
+const fetch = require('node-fetch');
+const ProgressBar = require('progress');
+
+const config_api = require('./config');
+const logger = require('./logger');
+const CONSTS = require('./consts');
+
+const is_windows = process.platform === 'win32';
+
+// replaces .webm with appropriate extension
+function getTrueFileName(unfixed_path, type) {
+    let fixed_path = unfixed_path;
+
+    const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
+    let unfixed_parts = unfixed_path.split('.');
+    const old_ext = unfixed_parts[unfixed_parts.length-1];
+
+
+    if (old_ext !== new_ext) {
+        unfixed_parts[unfixed_parts.length-1] = new_ext;
+        fixed_path = unfixed_parts.join('.');
+    }
+    return fixed_path;
+}
+
+async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
+    // return empty array if the path doesn't exist
+    if (!(await fs.pathExists(basePath))) return [];
+
+    let files = [];
+    const ext = type === 'audio' ? 'mp3' : 'mp4';
+    var located_files = await recFindByExt(basePath, ext);
+    for (let i = 0; i < located_files.length; i++) {
+        let file = located_files[i];
+        var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
+
+        var stats = await fs.stat(file);
+
+        var id = file_path.substring(0, file_path.length-4);
+        var jsonobj = await getJSONByType(type, id, basePath);
+        if (!jsonobj) continue;
+        if (full_metadata) {
+            jsonobj['id'] = id;
+            files.push(jsonobj);
+            continue;
+        }
+        var upload_date = formatDateString(jsonobj.upload_date);
+
+        var isaudio = type === 'audio';
+        var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
+                                stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
+        files.push(file_obj);
+    }
+    return files;
+}
+
+async function createContainerZipFile(file_name, container_file_objs) {
+    const container_files_to_download = [];
+    for (let i = 0; i < container_file_objs.length; i++) {
+        const container_file_obj = container_file_objs[i];
+        container_files_to_download.push(container_file_obj.path);
+    }
+    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+}
+
+async function createZipFile(zip_file_path, file_paths) {
+    let output = fs.createWriteStream(zip_file_path);
+
+    var archive = archiver('zip', {
+        gzip: true,
+        zlib: { level: 9 } // Sets the compression level.
+    });
+
+    archive.on('error', function(err) {
+        logger.error(err);
+        throw err;
+    });
+
+    // pipe archive data to the output file
+    archive.pipe(output);
+
+    for (let file_path of file_paths) {
+        const file_name = path.parse(file_path).base;
+        archive.file(file_path, {name: file_name})
+    }
+
+    await archive.finalize();
+
+    // wait a tiny bit for the zip to reload in fs
+    await wait(100);
+    return zip_file_path;
+}
+
+function getJSONMp4(name, customPath, openReadPerms = false) {
+    var obj = null; // output
+    if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
+    var jsonPath = path.join(customPath, name + ".info.json");
+    var alternateJsonPath = path.join(customPath, name + ".mp4.info.json");
+    if (fs.existsSync(jsonPath))
+    {
+        obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+    } else if (fs.existsSync(alternateJsonPath)) {
+        obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
+    }
+    else obj = 0;
+    return obj;
+}
+
+function getJSONMp3(name, customPath, openReadPerms = false) {
+    var obj = null;
+    if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
+    var jsonPath = path.join(customPath, name + ".info.json");
+    var alternateJsonPath = path.join(customPath, name + ".mp3.info.json");
+    if (fs.existsSync(jsonPath)) {
+        obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+    }
+    else if (fs.existsSync(alternateJsonPath)) {
+        obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
+    }
+    else
+        obj = 0;
+
+    return obj;
+}
+
+function getJSON(file_path, type) {
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+    let obj = null;
+    var jsonPath = removeFileExtension(file_path) + '.info.json';
+    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    if (fs.existsSync(jsonPath))
+    {
+        obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+    } else if (fs.existsSync(alternateJsonPath)) {
+        obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
+    }
+    else obj = 0;
+    return obj;
+}
+
+function getJSONByType(type, name, customPath, openReadPerms = false) {
+    return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
+}
+
+function getDownloadedThumbnail(file_path) {
+    const file_path_no_extension = removeFileExtension(file_path);
+
+    let jpgPath = file_path_no_extension + '.jpg';
+    let webpPath = file_path_no_extension + '.webp';
+    let pngPath = file_path_no_extension + '.png';
+
+    if (fs.existsSync(jpgPath))
+        return jpgPath;
+    else if (fs.existsSync(webpPath))
+        return webpPath;
+    else if (fs.existsSync(pngPath))
+        return pngPath;
+    else
+        return null;
+}
+
+function getExpectedFileSize(input_info_jsons) {
+    // treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
+    const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
+
+    let expected_filesize = 0;
+    info_jsons.forEach(info_json => {
+        const formats = info_json['format_id'].split('+');
+        let individual_expected_filesize = 0;
+        formats.forEach(format_id => {
+            if (info_json.formats !== undefined) {
+                info_json.formats.forEach(available_format => {
+                  if (available_format.format_id === format_id && (available_format.filesize || available_format.filesize_approx)) {
+                    individual_expected_filesize += (available_format.filesize ? available_format.filesize : available_format.filesize_approx);
+                  }
+                });
+            }
+        });
+        expected_filesize += individual_expected_filesize;
+    });
+
+    return expected_filesize;
+}
+
+function fixVideoMetadataPerms(file_path, type) {
+    if (is_windows) return;
+
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+
+    const file_path_no_extension = removeFileExtension(file_path);
+
+    const files_to_fix = [
+        // JSONs
+        file_path_no_extension + '.info.json',
+        file_path_no_extension + ext + '.info.json',
+        // Thumbnails
+        file_path_no_extension + '.webp',
+        file_path_no_extension + '.jpg'
+    ];
+
+    for (const file of files_to_fix) {
+        if (!fs.existsSync(file)) continue;
+        fs.chmodSync(file, 0o644);
+    }
+}
+
+function deleteJSONFile(file_path, type) {
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+
+    const file_path_no_extension = removeFileExtension(file_path);
+
+    let json_path = file_path_no_extension + '.info.json';
+    let alternate_json_path = file_path_no_extension + ext + '.info.json';
+
+    if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
+    if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
+}
+
+// archive helper functions
+
+async function removeIDFromArchive(archive_path, type, id) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
+    if (!data) {
+        logger.error('Archive could not be found.');
+        return;
+    }
+
+    let dataArray = data.split('\n'); // convert file data in an array
+    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
+    let lastIndex = -1; // let say, we have not found the keyword
+
+    for (let index=0; index<dataArray.length; index++) {
+        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+            lastIndex = index; // found a line includes a id keyword
+            break;
+        }
+    }
+
+    if (lastIndex === -1) return null;
+
+    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
+
+    // UPDATE FILE WITH NEW DATA
+    const updatedData = dataArray.join('\n');
+    await fs.writeFile(archive_file, updatedData);
+    if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
+}
+
+async function writeToBlacklist(archive_folder, type, line) {
+    let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
+    // adds newline to the beginning of the line
+    line.replace('\n', '');
+    line.replace('\r', '');
+    line = '\n' + line;
+    await fs.appendFile(blacklistPath, line);
+}
+
+async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    if (await fs.pathExists(archive_path)) {
+        const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
+        if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
+    } else {
+        logger.info(`Could not find archive file for file ${uid}. Creating...`);
+        await fs.close(await fs.open(archive_file, 'w'));
+    }
+}
+
+function durationStringToNumber(dur_str) {
+    if (typeof dur_str === 'number') return dur_str;
+    let num_sum = 0;
+    const dur_str_parts = dur_str.split(':');
+    for (let i = dur_str_parts.length-1; i >= 0; i--) {
+      num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
+    }
+    return num_sum;
+}
+
+function getMatchingCategoryFiles(category, files) {
+    return files && files.filter(file => file.category && file.category.uid === category.uid);
+}
+
+function addUIDsToCategory(category, files) {
+    const files_that_match = getMatchingCategoryFiles(category, files);
+    category['uids'] = files_that_match.map(file => file.uid);
+    return files_that_match;
+}
+
+function getCurrentDownloader() {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    return details_json['downloader'];
+}
+
+async function recFindByExt(base, ext, files, result, recursive = true)
+{
+    files = files || (await fs.readdir(base))
+    result = result || []
+
+    for (const file of files) {
+        var newbase = path.join(base,file)
+        if ( (await fs.stat(newbase)).isDirectory() )
+        {
+            if (!recursive) continue;
+            result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
+        }
+        else
+        {
+            if ( file.substr(-1*(ext.length+1)) == '.' + ext )
+            {
+                result.push(newbase)
+            }
+        }
+    }
+    return result
+}
+
+function removeFileExtension(filename) {
+    const filename_parts = filename.split('.');
+    filename_parts.splice(filename_parts.length - 1);
+    return filename_parts.join('.');
+}
+
+function formatDateString(date_string) {
+    return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
+}
+
+function createEdgeNGrams(str) {
+    if (str && str.length > 3) {
+        const minGram = 3
+        const maxGram = str.length
+
+        return str.split(" ").reduce((ngrams, token) => {
+            if (token.length > minGram) {
+                for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
+                    ngrams = [...ngrams, token.substr(0, i)]
+                }
+            } else {
+                ngrams = [...ngrams, token]
+            }
+            return ngrams
+        }, []).join(" ")
+    }
+
+    return str
+}
+
+// ffmpeg helper functions
+
+async function cropFile(file_path, start, end, ext) {
+    return new Promise(resolve => {
+        const temp_file_path = `${file_path}.cropped${ext}`;
+        let base_ffmpeg_call = ffmpeg(file_path);
+        if (start) {
+            base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
+        }
+        if (end) {
+            base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
+        }
+        base_ffmpeg_call
+            .on('end', () => {
+                logger.verbose(`Cropping for '${file_path}' complete.`);
+                fs.unlinkSync(file_path);
+                fs.moveSync(temp_file_path, file_path);
+                resolve(true);
+            })
+            .on('error', (err) => {
+                logger.error(`Failed to crop ${file_path}.`);
+                logger.error(err);
+                resolve(false);
+            }).save(temp_file_path);
+    });
+}
+
+/**
+ * setTimeout, but its a promise.
+ * @param {number} ms
+ */
+ async function wait(ms) {
+    await new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+
+async function checkExistsWithTimeout(filePath, timeout) {
+    return new Promise(function (resolve, reject) {
+
+        var timer = setTimeout(function () {
+            if (watcher) watcher.close();
+            reject(new Error('File did not exists and was not created during the timeout.'));
+        }, timeout);
+
+        fs.access(filePath, fs.constants.R_OK, function (err) {
+            if (!err) {
+                clearTimeout(timer);
+                if (watcher) watcher.close();
+                resolve();
+            }
+        });
+
+        var dir = path.dirname(filePath);
+        var basename = path.basename(filePath);
+        var watcher = fs.watch(dir, function (eventType, filename) {
+            if (eventType === 'rename' && filename === basename) {
+                clearTimeout(timer);
+                if (watcher) watcher.close();
+                resolve();
+            }
+        });
+    });
+}
+
+// helper function to download file using fetch
+async function fetchFile(url, path, file_label) {
+    var len = null;
+    const res = await fetch(url);
+
+    len = parseInt(res.headers.get("Content-Length"), 10);
+
+    var bar = new ProgressBar(`  Downloading ${file_label} [:bar] :percent :etas`, {
+        complete: '=',
+        incomplete: ' ',
+        width: 20,
+        total: len
+    });
+    const fileStream = fs.createWriteStream(path);
+    await new Promise((resolve, reject) => {
+        res.body.pipe(fileStream);
+        res.body.on("error", (err) => {
+          reject(err);
+        });
+        res.body.on('data', function (chunk) {
+            bar.tick(chunk.length);
+        });
+        fileStream.on("finish", function() {
+          resolve();
+        });
+    });
+}
+
+async function restartServer(is_update = false) {
+    logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
+
+    // the following line restarts the server through pm2
+    fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
+    process.exit(1);
+}
+
+// adds or replaces args according to the following rules:
+//  - if it already exists and has value, then replace both arg and value
+//  - if already exists and doesn't have value, ignore
+//  - if it doesn't exist and has value, add both arg and value
+//  - if it doesn't exist and doesn't have value, add arg
+function injectArgs(original_args, new_args) {
+    const updated_args = original_args.slice();
+    try {
+        for (let i = 0; i < new_args.length; i++) {
+            const new_arg = new_args[i];
+            if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
+
+            if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
+                if (original_args.includes(new_arg)) {
+                    const original_index = original_args.indexOf(new_arg);
+                    original_args.splice(original_index, 2);
+                }
+
+                updated_args.push(new_arg, new_args[i + 1]);
+            } else {
+                if (!original_args.includes(new_arg)) {
+                    updated_args.push(new_arg);
+                }
+            }
+        }
+    } catch (err) {
+        logger.warn(err);
+        logger.warn(`Failed to inject args (${new_args}) into (${original_args})`);
+    }
+
+    return updated_args;
+}
+
+function filterArgs(args, args_to_remove) {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+const searchObjectByString = function(o, s) {
+    s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
+    s = s.replace(/^\./, '');           // strip a leading dot
+    var a = s.split('.');
+    for (var i = 0, n = a.length; i < n; ++i) {
+        var k = a[i];
+        if (k in o) {
+            o = o[k];
+        } else {
+            return;
+        }
+    }
+    return o;
+}
+
+function stripPropertiesFromObject(obj, properties, whitelist = false) {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
+function getArchiveFolder(type, user_uid = null, sub = null) {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
+// objects
+
+function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
+    this.id = id;
+    this.title = title;
+    this.thumbnailURL = thumbnailURL;
+    this.isAudio = isAudio;
+    this.duration = duration;
+    this.url = url;
+    this.uploader = uploader;
+    this.size = size;
+    this.path = path;
+    this.upload_date = upload_date;
+    this.description = description;
+    this.view_count = view_count;
+    this.height = height;
+    this.abr = abr;
+}
+
+module.exports = {
+    getJSONMp3: getJSONMp3,
+    getJSONMp4: getJSONMp4,
+    getJSON: getJSON,
+    getTrueFileName: getTrueFileName,
+    getDownloadedThumbnail: getDownloadedThumbnail,
+    getExpectedFileSize: getExpectedFileSize,
+    fixVideoMetadataPerms: fixVideoMetadataPerms,
+    deleteJSONFile: deleteJSONFile,
+    removeIDFromArchive: removeIDFromArchive,
+    writeToBlacklist: writeToBlacklist,
+    deleteFileFromArchive: deleteFileFromArchive,
+    getDownloadedFilesByType: getDownloadedFilesByType,
+    createContainerZipFile: createContainerZipFile,
+    durationStringToNumber: durationStringToNumber,
+    getMatchingCategoryFiles: getMatchingCategoryFiles,
+    getCurrentDownloader: getCurrentDownloader,
+    recFindByExt: recFindByExt,
+    removeFileExtension: removeFileExtension,
+    formatDateString: formatDateString,
+    cropFile: cropFile,
+    createEdgeNGrams: createEdgeNGrams,
+    wait: wait,
+    checkExistsWithTimeout: checkExistsWithTimeout,
+    fetchFile: fetchFile,
+    restartServer: restartServer,
+    injectArgs: injectArgs,
+    filterArgs: filterArgs,
+    searchObjectByString: searchObjectByString,
+    stripPropertiesFromObject: stripPropertiesFromObject,
+    getArchiveFolder: getArchiveFolder,
+    File: File
+}

backend/video/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

backend/youtube-dl.js

@@ -0,0 +1,141 @@
+const fs = require('fs-extra');
+const fetch = require('node-fetch');
+
+const logger = require('./logger');
+const utils = require('./utils');
+const CONSTS = require('./consts');
+const config_api = require('./config.js');
+
+const OUTDATED_VERSION = "2020.00.00";
+
+const is_windows = process.platform === 'win32';
+
+const download_sources = {
+    'youtube-dl': {
+        'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
+        'func': downloadLatestYoutubeDLBinary
+    },
+    'youtube-dlc': {
+        'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
+        'func': downloadLatestYoutubeDLCBinary
+    },
+    'yt-dlp': {
+        'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
+        'func': downloadLatestYoutubeDLPBinary
+    }
+}
+
+exports.checkForYoutubeDLUpdate = async () => {
+    return new Promise(async resolve => {
+        const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+        const tags_url = download_sources[default_downloader]['tags_url'];
+        // get current version
+        let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
+        if (!current_app_details_exists) {
+            logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
+            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader});
+        }
+        let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
+        let current_version = current_app_details['version'];
+        let current_downloader = current_app_details['downloader'];
+        let stored_binary_path = current_app_details['path'];
+        if (!stored_binary_path || typeof stored_binary_path !== 'string') {
+            // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
+            const guessed_base_path = 'node_modules/youtube-dl/bin/';
+            const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
+            if (fs.existsSync(guessed_file_path)) {
+                stored_binary_path = guessed_file_path;
+                // logger.info('INFO: Guess successful! Update process continuing...')
+            } else {
+                logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
+                resolve(null);
+                return;
+            }
+        }
+
+        // got version, now let's check the latest version from the youtube-dl API
+
+
+        fetch(tags_url, {method: 'Get'})
+        .then(async res => res.json())
+        .then(async (json) => {
+            // check if the versions are different
+            if (!json || !json[0]) {
+                logger.error(`Failed to check ${default_downloader} version for an update.`)
+                resolve(null);
+                return;
+            }
+            const latest_update_version = json[0]['name'];
+            if (current_version !== latest_update_version || default_downloader !== current_downloader) {
+                // versions different or different downloader is being used, download new update
+                resolve(latest_update_version);
+            } else {
+                resolve(null);
+            }
+            return;
+        })
+        .catch(err => {
+            logger.error(`Failed to check ${default_downloader} version for an update.`)
+            logger.error(err);
+            resolve(null);
+            return;
+        });
+    });
+}
+
+exports.updateYoutubeDL = async (latest_update_version) => {
+    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
+    await download_sources[default_downloader]['func'](latest_update_version);
+}
+
+exports.verifyBinaryExistsLinux = () => {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
+        details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
+        details_json['exec'] = 'youtube-dl';
+        details_json['version'] = OUTDATED_VERSION;
+        fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
+
+        utils.restartServer();
+    }
+}
+
+async function downloadLatestYoutubeDLBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
+
+    updateDetailsJSON(new_version, 'youtube-dl');
+}
+
+async function downloadLatestYoutubeDLCBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
+
+    updateDetailsJSON(new_version, 'youtube-dlc');
+}
+
+async function downloadLatestYoutubeDLPBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
+
+    updateDetailsJSON(new_version, 'yt-dlp');
+}
+
+function updateDetailsJSON(new_version, downloader) {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (new_version) details_json['version'] = new_version;
+    details_json['downloader'] = downloader;
+    fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
+}

chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: youtubedl-material
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "4.3"

chart/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "youtubedl-material.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "youtubedl-material.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "youtubedl-material.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "youtubedl-material.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

chart/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "youtubedl-material.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "youtubedl-material.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "youtubedl-material.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "youtubedl-material.labels" -}}
+helm.sh/chart: {{ include "youtubedl-material.chart" . }}
+{{ include "youtubedl-material.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "youtubedl-material.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "youtubedl-material.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "youtubedl-material.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "youtubedl-material.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

chart/templates/appdata-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.appdata.enabled (not .Values.persistence.appdata.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-appdata
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.appdata.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.appdata.size | quote }}
+  {{- if .Values.persistence.appdata.storageClass }}
+  {{- if (eq "-" .Values.persistence.appdata.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.appdata.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/audio-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.audio.enabled (not .Values.persistence.audio.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-audio
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.audio.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.audio.size | quote }}
+  {{- if .Values.persistence.audio.storageClass }}
+  {{- if (eq "-" .Values.persistence.audio.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.audio.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/deployment.yaml

@@ -0,0 +1,121 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "youtubedl-material.fullname" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "youtubedl-material.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "youtubedl-material.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "youtubedl-material.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 17442
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /app/appdata
+              name: appdata
+              {{- if .Values.persistence.appdata.subPath }}
+              subPath: {{ .Values.persistence.appdata.subPath }}
+              {{- end }}
+            - mountPath: /app/audio
+              name: audio
+              {{- if .Values.persistence.audio.subPath }}
+              subPath: {{ .Values.persistence.audio.subPath }}
+              {{- end }}
+            - mountPath: /app/video
+              name: video
+              {{- if .Values.persistence.video.subPath }}
+              subPath: {{ .Values.persistence.video.subPath }}
+              {{- end }}
+            - mountPath: /app/subscriptions
+              name: subscriptions
+              {{- if .Values.persistence.subscriptions.subPath }}
+              subPath: {{ .Values.persistence.subscriptions.subPath }}
+              {{- end }}
+            - mountPath: /app/users
+              name: users
+              {{- if .Values.persistence.users.subPath }}
+              subPath: {{ .Values.persistence.users.subPath }}
+              {{- end }}
+      volumes:
+        - name: appdata
+          {{- if .Values.persistence.appdata.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.appdata.existingClaim }}{{ .Values.persistence.appdata.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-appdata{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: audio
+          {{- if .Values.persistence.audio.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.audio.existingClaim }}{{ .Values.persistence.audio.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-audio{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: subscriptions
+          {{- if .Values.persistence.subscriptions.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.subscriptions.existingClaim }}{{ .Values.persistence.subscriptions.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-subscriptions{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: users
+          {{- if .Values.persistence.users.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.users.existingClaim }}{{ .Values.persistence.users.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-users{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: video
+          {{- if .Values.persistence.video.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.video.existingClaim }}{{ .Values.persistence.video.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-video{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

chart/templates/ingress.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "youtubedl-material.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+          {{- end }}
+    {{- end }}
+  {{- end }}

chart/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "youtubedl-material.fullname" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "youtubedl-material.selectorLabels" . | nindent 4 }}

chart/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "youtubedl-material.serviceAccountName" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/templates/subscriptions-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.subscriptions.enabled (not .Values.persistence.subscriptions.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-subscriptions
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.subscriptions.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.subscriptions.size | quote }}
+  {{- if .Values.persistence.subscriptions.storageClass }}
+  {{- if (eq "-" .Values.persistence.subscriptions.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.subscriptions.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "youtubedl-material.fullname" . }}-test-connection"
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "youtubedl-material.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

chart/templates/users-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.users.enabled (not .Values.persistence.users.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-users
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.users.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.users.size | quote }}
+  {{- if .Values.persistence.users.storageClass }}
+  {{- if (eq "-" .Values.persistence.users.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.users.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/video-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.video.enabled (not .Values.persistence.video.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-video
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.video.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.video.size | quote }}
+  {{- if .Values.persistence.video.storageClass }}
+  {{- if (eq "-" .Values.persistence.video.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.video.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/values.yaml

@@ -0,0 +1,153 @@
+# Default values for youtubedl-material.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: tzahi12345/youtubedl-material
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 17442
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+persistence:
+  appdata:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 1Gi
+  audio:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  video:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  subscriptions:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  users:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

chrome-extension/manifest.json

@@ -0,0 +1,26 @@
+{
+    "manifest_version": 2,
+    "name": "YoutubeDL-Material",
+    "version": "0.4",
+    "description": "The Official Firefox & Chrome Extension of YoutubeDL-Material, an open-source and self-hosted YouTube downloader.",
+    "browser_action": {
+        "default_icon": "favicon.png",
+        "default_popup": "popup.html",
+        "default_title": "YoutubeDL-Material"
+    },
+    "permissions": [
+        "tabs",
+        "storage",
+        "contextMenus"
+    ],
+    "options_ui": {
+        "page": "options.html",
+        "open_in_tab": false
+    },
+    "browser_specific_settings": {
+        "gecko": {
+          "id": "IsaacMGrynsztein@gmail.com",
+          "strict_min_version": "57.0"
+        }
+    }
+}

chrome-extension/options.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>YoutubeDL-Material Extension Options</title>
+    <!-- Scripts -->
+    <script src="js/jquery-3.4.1.min.js"></script>
+    <script src="js/popper.min.js"></script>
+    <script src="js/bootstrap.min.js"></script>
+    
+    <!-- Cascading Style Sheets -->
+    <link href="css/bootstrap.min.css" rel="stylesheet" media="screen">
+</head>
+<body>
+
+<h2>Settings</h2>
+
+<div style="width: 95%; margin: 0 auto;">
+    <div class="form-group">
+    <label for="frontend_url">Frontend URL:</label>
+    <input class="form-control" type="text" id="frontend_url">
+    </div>
+
+    <div class="checkbox">
+        <label>
+            <input type="checkbox" id="audio_only">
+            Audio only
+        </label>
+    </div>
+
+    <div id="status"></div>
+    <button style="margin-bottom: 10px;" class="btn btn-primary" data-toggle="collapse" data-target="#collapseExample" aria-expanded="false" aria-controls="collapseExample" id="save">Save</button>
+
+    <div class="collapse" id="collapseExample">
+        <div class="card card-body">
+        Settings successfully saved!
+        </div>
+    </div>
+</div>
+
+<script src="options.js"></script>
+</body>
+</html>

chrome-extension/options.js

@@ -0,0 +1,30 @@
+// Saves options to chrome.storage
+function save_options() {
+    var frontend_url = document.getElementById('frontend_url').value;
+    var audio_only = document.getElementById('audio_only').checked;
+    chrome.storage.sync.set({
+        frontend_url: frontend_url,
+        audio_only: audio_only
+    }, function() {
+      // Update status to let user know options were saved.
+      $('#collapseExample').collapse('show');
+      setTimeout(function() {
+        $('#collapseExample').collapse('hide');
+      }, 2000);
+    });
+  }
+  
+  // Restores select box and checkbox state using the preferences
+  // stored in chrome.storage.
+  function restore_options() {
+    chrome.storage.sync.get({
+        frontend_url: 'http://localhost',
+        audio_only: false
+    }, function(items) {
+      document.getElementById('frontend_url').value = items.frontend_url;
+      document.getElementById('audio_only').checked = items.audio_only;
+    });
+  }
+  document.addEventListener('DOMContentLoaded', restore_options);
+  document.getElementById('save').addEventListener('click',
+      save_options);

chrome-extension/popup.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <!-- Scripts -->
+    <script src="js/jquery-3.4.1.min.js"></script>
+    <script src="js/popper.min.js"></script>
+    <script src="js/bootstrap.min.js"></script>
+    
+    <!-- Cascading Style Sheets -->
+    <link href="css/bootstrap.min.css" rel="stylesheet" media="screen">
+</head>
+<body>
+
+<div style="width: 400px; margin: 0 auto;">
+    <div style="margin: 10px;">
+        <div class="checkbox">
+            <label>
+                <input type="checkbox" id="audio_only">
+                Audio only
+            </label>
+        </div>
+
+        <div class="input-group mb-3">
+            <input id="url_input" type="text" class="form-control" placeholder="URL" aria-label="URL" aria-describedby="basic-addon2">
+            <div class="input-group-append">
+            <button class="btn btn-outline-secondary" type="button" id="download">Download</button>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script src="popup.js"></script>
+
+</body>
+</html>

chrome-extension/popup.js

@@ -0,0 +1,50 @@
+function audioOnlyClicked() {
+    console.log('audio only clicked');
+    var audio_only = document.getElementById("audio_only").checked;
+
+    // save state
+
+    chrome.storage.sync.set({
+        audio_only: audio_only
+    }, function() {});
+}
+
+function downloadVideo() {
+    var input_url = document.getElementById("url_input").value
+    // get the frontend_url
+    chrome.storage.sync.get({
+        frontend_url: 'http://localhost',
+        audio_only: false
+    }, function(items) {
+        var download_url = items.frontend_url + '/#/home;url=' + encodeURIComponent(input_url) + ';audioOnly=' + items.audio_only;
+        chrome.tabs.create({ url: download_url });
+    });
+}
+
+function loadInputs() {
+    // load audio-only input
+    chrome.storage.sync.get({
+        frontend_url: 'http://localhost',
+        audio_only: false
+    }, function(items) {
+        document.getElementById("audio_only").checked = items.audio_only;
+    });
+    
+    // load url input
+    chrome.tabs.query({active: true, currentWindow: true}, function(tabs) {
+        var activeTab = tabs[0];
+        var current_url = activeTab.url;
+        console.log(current_url);
+        if (current_url && current_url.includes('youtube.com')) {
+            document.getElementById("url_input").value = current_url;
+        }
+    });
+}
+
+document.getElementById('download').addEventListener('click',
+      downloadVideo);
+
+document.getElementById('audio_only').addEventListener('click',
+      audioOnlyClicked);
+
+document.addEventListener('DOMContentLoaded', loadInputs);

db.json

@@ -1,6 +0,0 @@
-{
-  "playlists": {
-    "audio": [],
-    "video": []
-  }
-}

docker-compose.yml

@@ -1,35 +1,30 @@
-
 version: "2"
 services:
     ytdl_material:
-        build: .
         environment: 
-            # config items
-            ytdl_frontend_url: http://localhost:8998
-            ytdl_backend_url: http://localhost:17442/
-            ytdl_use_encryption: 'false'
-            ytdl_cert_file_path: /etc/letsencrypt/live/example.com/fullchain.pem
-            ytdl_key_file_path: /etc/letsencrypt/live/example.com/privkey.pem
-            ytdl_base_path: http://localhost:17442/
-            ytdl_audio_folder_path: audio/
-            ytdl_video_folder_path: video/
-            ytdl_title_top: Youtube Downloader
-            ytdl_file_manager_enabled: 'true'
-            ytdl_allow_quality_select: 'true'
-            ytdl_download_only_mode: 'false'
-            ytdl_allow_multi_download_mode: true
-            ytdl_use_youtube_api: 'false'
-            ytdl_youtube_api_key: 'false'
-            ytdl_default_theme: default
-            ytdl_allow_theme_change: 'true'
-            ytdl_use_default_downloading_agent: 'true'
-            ytdl_custom_downloading_agent: 'false'
-            ytdl_allow_advanced_download: 'false'
-            # do not touch this
-            write_ytdl_config: 'true'
             ALLOW_CONFIG_MUTATIONS: 'true'
+            ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
+            ytdl_use_local_db: 'false'
+            write_ytdl_config: 'true'
         restart: always
+        depends_on:
+            - ytdl-mongo-db
+        volumes:
+            - ./appdata:/app/appdata
+            - ./audio:/app/audio
+            - ./video:/app/video
+            - ./subscriptions:/app/subscriptions
+            - ./users:/app/users
         ports:
-            - "17442:17442"
-            - "8998:80"
-        image: tzahi12345/youtubedl-material:3.1
+            - "8998:17442"
+        image: tzahi12345/youtubedl-material:latest
+    ytdl-mongo-db:
+        image: mongo
+        ports:
+            - "27017:27017"
+        logging:
+            driver: "none"          
+        container_name: mongo-db
+        restart: always
+        volumes:
+            - ./db/:/data/db

docker_wrapper.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-cd backend
-
-# Start the first process
-node app.js &
-status=$?
-if [ $status -ne 0 ]; then
-  echo "Failed to start my_first_process: $status"
-  exit $status
-fi
-
-# Start the second process
-apachectl -DFOREGROUND
-status=$?
-if [ $status -ne 0 ]; then
-  echo "Failed to start my_second_process: $status"
-  exit $status
-fi
-
-# Naive check runs checks once a minute to see if either of the processes exited.
-# This illustrates part of the heavy lifting you need to do if you want to run
-# more than one service in a container. The container will exit with an error
-# if it detects that either of the processes has exited.
-# Otherwise it will loop forever, waking up every 60 seconds
-
-while /bin/true; do
-  ps aux |grep node\ app.js # |grep -q -v grep
-  PROCESS_1_STATUS=$?
-  ps aux |grep apache2 # |grep -q -v grep
-  PROCESS_2_STATUS=$?
-  # If the greps above find anything, they will exit with 0 status
-  # If they are not both 0, then something is wrong
-  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
-    echo "One of the processes has already exited."
-    exit -1
-  fi
-  sleep 60
-done

ffmpeg-fetch.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
+# and also optimizing some code with this commit.
+# xoxo :D
+
+case $(uname -m) in
+  x86_64)
+    ARCH=amd64;;
+  aarch64)
+    ARCH=arm64;;
+  armhf)
+    ARCH=armhf;;
+  armv7)
+    ARCH=armel;;
+  armv7l)
+    ARCH=armel;;
+  *)
+    echo "Unsupported architecture: $(uname -m)"
+    exit 1
+esac
+
+echo "(INFO) Architecture detected: $ARCH"
+echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
+apt-get update && apt-get -y install curl xz-utils
+echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
+curl -o ffmpeg.txz \
+    --connect-timeout 5 \
+    --max-time 10 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
+mkdir /tmp/ffmpeg
+tar xf ffmpeg.txz -C /tmp/ffmpeg
+echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
+apt-get -y remove curl xz-utils
+apt-get -y autoremove
+echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
+cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
+cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
+echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
+rm -rf /tmp/ffmpeg ffmpeg.txz

heroku.yml

@@ -0,0 +1,3 @@
+build:
+  docker:
+    web: Dockerfile.heroku

package.json

@@ -1,15 +1,19 @@
 {
   "name": "youtube-dl-material",
-  "version": "0.0.0",
+  "version": "4.3.0",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
-    "build": "ng build",
+    "build": "ng build --configuration production",
+    "prebuild": "node src/postbuild.mjs",
+    "heroku-postbuild": "npm install --prefix backend",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e",
-    "electron": "ng build --base-href ./ && electron ."
+    "electron": "ng build --base-href ./ && electron .",
+    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n"
   },
   "engines": {
     "node": "12.3.1",
@@ -17,53 +21,64 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^8.3.12",
-    "@angular/animations": "^8.2.11",
-    "@angular/cdk": "^8.2.3",
-    "@angular/common": "^8.2.11",
-    "@angular/compiler": "^8.2.11",
-    "@angular/core": "^8.2.11",
-    "@angular/forms": "^8.2.11",
-    "@angular/http": "^7.2.15",
-    "@angular/material": "^8.2.3",
-    "@angular/platform-browser": "^8.2.11",
-    "@angular/platform-browser-dynamic": "^8.2.11",
-    "@angular/router": "^8.2.11",
+    "@angular-devkit/core": "^13.3.3",
+    "@angular/animations": "^13.3.4",
+    "@angular/cdk": "^13.3.4",
+    "@angular/common": "^13.3.4",
+    "@angular/compiler": "^13.3.4",
+    "@angular/core": "^13.3.4",
+    "@angular/forms": "^13.3.4",
+    "@angular/localize": "^13.3.4",
+    "@angular/material": "^13.3.4",
+    "@angular/platform-browser": "^13.3.4",
+    "@angular/platform-browser-dynamic": "^13.3.4",
+    "@angular/router": "^13.3.4",
+    "@fontsource/material-icons": "^4.5.4",
+    "@ngneat/content-loader": "^5.0.0",
+    "@videogular/ngx-videogular": "^5.0.1",
     "core-js": "^2.4.1",
+    "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
+    "filesize": "^6.1.0",
+    "fingerprintjs2": "^2.1.0",
+    "fs-extra": "^10.0.0",
+    "material-icons": "^1.10.8",
+    "nan": "^2.14.1",
     "ng-lazyload-image": "^7.0.1",
-    "ng4-configure": "^0.1.7",
-    "ngx-content-loading": "^0.1.3",
-    "rxjs": "^6.5.3",
+    "ngx-avatars": "^1.3.1",
+    "ngx-file-drop": "^13.0.0",
+    "rxjs": "^6.6.3",
     "rxjs-compat": "^6.0.0-rc.0",
-    "tslib": "^1.10.0",
-    "videogular2": "^7.0.1",
-    "web-animations-js": "^2.3.2",
-    "zone.js": "~0.9.1",
-    "typescript": "~3.5.3"
+    "tslib": "^2.0.0",
+    "typescript": "~4.6.3",
+    "xliff-to-json": "^1.0.4",
+    "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.803.24",
-    "@angular/cli": "^8.3.12",
-    "@angular/compiler-cli": "^8.2.11",
-    "@angular/language-service": "^8.2.11",
+    "@angular-devkit/build-angular": "^13.3.3",
+    "@angular/cli": "^13.3.3",
+    "@angular/compiler-cli": "^13.3.4",
+    "@angular/language-service": "^13.3.4",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
-    "@types/jasmine": "2.5.45",
-    "@types/node": "~6.0.60",
-    "codelyzer": "^5.0.1",
-    "electron": "^8.0.1",
-    "jasmine-core": "~2.6.2",
-    "jasmine-spec-reporter": "~4.1.0",
-    "karma": "~1.7.0",
-    "karma-chrome-launcher": "~2.1.1",
+    "@types/jasmine": "~3.6.0",
+    "@types/node": "^12.11.1",
+    "@typescript-eslint/eslint-plugin": "^4.29.0",
+    "@typescript-eslint/parser": "^4.29.0",
+    "codelyzer": "^6.0.0",
+    "electron": "^19.0.6",
+    "eslint": "^7.32.0",
+    "jasmine-core": "~3.6.0",
+    "jasmine-spec-reporter": "~5.0.0",
+    "karma": "~6.3.16",
+    "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
-    "karma-coverage-istanbul-reporter": "^1.2.1",
-    "karma-jasmine": "~1.1.0",
-    "karma-jasmine-html-reporter": "^0.2.2",
-    "protractor": "~5.1.2",
+    "karma-coverage-istanbul-reporter": "~3.0.2",
+    "karma-jasmine": "~4.0.0",
+    "karma-jasmine-html-reporter": "^1.5.0",
+    "openapi-typescript-codegen": "^0.21.0",
+    "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
-    "tslint": "~5.3.2",
-    "typescript": "~3.5.3"
+    "tslint": "~6.1.0"
   }
 }

src/_palette.scss

@@ -1,11 +1,11 @@
 /* Coolors Exported Palette - coolors.co/e8aeb7-b8e1ff-a9fff7-94fbab-82aba1 */
 
 /* HSL */
-$color1: hsla(351%, 56%, 80%, 1);
-$softblue: hsla(205%, 100%, 86%, 1);
-$color3: hsla(174%, 100%, 83%, 1);
-$color4: hsla(133%, 93%, 78%, 1);
-$color5: hsla(165%, 20%, 59%, 1);
+$color1: hsla(351, 56%, 80%, 1);
+$softblue: hsla(205, 100%, 86%, 1);
+$color3: hsla(174, 100%, 83%, 1);
+$color4: hsla(133, 93%, 78%, 1);
+$color5: hsla(165, 20%, 59%, 1);
 
 /* RGB */
 $color1: rgba(232, 174, 183, 1);

src/api-types/index.ts

@@ -0,0 +1,117 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
+export type { binary } from './models/binary';
+export type { body_19 } from './models/body_19';
+export type { body_20 } from './models/body_20';
+export type { Category } from './models/Category';
+export { CategoryRule } from './models/CategoryRule';
+export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermissionsRequest';
+export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
+export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
+export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
+export type { ClearDownloadsRequest } from './models/ClearDownloadsRequest';
+export type { ConcurrentStream } from './models/ConcurrentStream';
+export type { Config } from './models/Config';
+export type { ConfigResponse } from './models/ConfigResponse';
+export type { CreateCategoryRequest } from './models/CreateCategoryRequest';
+export type { CreateCategoryResponse } from './models/CreateCategoryResponse';
+export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
+export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
+export type { CropFileSettings } from './models/CropFileSettings';
+export type { DatabaseFile } from './models/DatabaseFile';
+export { DBBackup } from './models/DBBackup';
+export type { DBInfoResponse } from './models/DBInfoResponse';
+export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
+export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
+export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
+export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
+export type { DeleteUserRequest } from './models/DeleteUserRequest';
+export type { Download } from './models/Download';
+export type { DownloadArchiveRequest } from './models/DownloadArchiveRequest';
+export type { DownloadFileRequest } from './models/DownloadFileRequest';
+export type { DownloadRequest } from './models/DownloadRequest';
+export type { DownloadResponse } from './models/DownloadResponse';
+export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
+export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
+export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
+export { FileType } from './models/FileType';
+export { FileTypeFilter } from './models/FileTypeFilter';
+export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
+export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
+export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
+export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
+export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
+export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
+export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
+export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
+export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
+export type { GetDownloadRequest } from './models/GetDownloadRequest';
+export type { GetDownloadResponse } from './models/GetDownloadResponse';
+export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
+export type { GetFileFormatsResponse } from './models/GetFileFormatsResponse';
+export type { GetFileRequest } from './models/GetFileRequest';
+export type { GetFileResponse } from './models/GetFileResponse';
+export type { GetFullTwitchChatRequest } from './models/GetFullTwitchChatRequest';
+export type { GetFullTwitchChatResponse } from './models/GetFullTwitchChatResponse';
+export type { GetLogsRequest } from './models/GetLogsRequest';
+export type { GetLogsResponse } from './models/GetLogsResponse';
+export type { GetMp3sResponse } from './models/GetMp3sResponse';
+export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
+export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
+export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
+export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
+export type { GetRolesResponse } from './models/GetRolesResponse';
+export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
+export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
+export type { GetTaskRequest } from './models/GetTaskRequest';
+export type { GetTaskResponse } from './models/GetTaskResponse';
+export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
+export type { inline_response_200_15 } from './models/inline_response_200_15';
+export type { LoginRequest } from './models/LoginRequest';
+export type { LoginResponse } from './models/LoginResponse';
+export type { Playlist } from './models/Playlist';
+export type { RegisterRequest } from './models/RegisterRequest';
+export type { RegisterResponse } from './models/RegisterResponse';
+export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
+export { Schedule } from './models/Schedule';
+export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SharingToggle } from './models/SharingToggle';
+export type { Sort } from './models/Sort';
+export type { SubscribeRequest } from './models/SubscribeRequest';
+export type { SubscribeResponse } from './models/SubscribeResponse';
+export type { Subscription } from './models/Subscription';
+export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
+export type { SuccessObject } from './models/SuccessObject';
+export type { TableInfo } from './models/TableInfo';
+export type { Task } from './models/Task';
+export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
+export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
+export type { TransferDBRequest } from './models/TransferDBRequest';
+export type { TransferDBResponse } from './models/TransferDBResponse';
+export type { TwitchChatMessage } from './models/TwitchChatMessage';
+export type { UnsubscribeRequest } from './models/UnsubscribeRequest';
+export type { UnsubscribeResponse } from './models/UnsubscribeResponse';
+export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
+export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
+export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
+export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
+export type { UpdateFileRequest } from './models/UpdateFileRequest';
+export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
+export type { UpdaterStatus } from './models/UpdaterStatus';
+export type { UpdateServerRequest } from './models/UpdateServerRequest';
+export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
+export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
+export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { User } from './models/User';
+export { UserPermission } from './models/UserPermission';
+export type { Version } from './models/Version';
+export type { VersionInfoResponse } from './models/VersionInfoResponse';
+export { YesNo } from './models/YesNo';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type AddFileToPlaylistRequest = {
+    file_uid: string;
+    playlist_id: string;
+};

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { UserPermission } from './UserPermission';
+import type { YesNo } from './YesNo';
+
+export type BaseChangePermissionsRequest = {
+    permission: UserPermission;
+    new_value: YesNo;
+};

src/api-types/models/Category.ts

@@ -0,0 +1,15 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { CategoryRule } from './CategoryRule';
+
+export type Category = {
+    name?: string;
+    uid?: string;
+    rules?: Array<CategoryRule>;
+    /**
+     * Overrides file output for downloaded files in category
+     */
+    custom_output?: string;
+};

src/api-types/models/CategoryRule.ts

@@ -0,0 +1,25 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type CategoryRule = {
+    preceding_operator?: CategoryRule.preceding_operator;
+    comparator?: CategoryRule.comparator;
+};
+
+export namespace CategoryRule {
+
+    export enum preceding_operator {
+        OR = 'or',
+        AND = 'and',
+    }
+
+    export enum comparator {
+        INCLUDES = 'includes',
+        NOT_INCLUDES = 'not_includes',
+        EQUALS = 'equals',
+        NOT_EQUALS = 'not_equals',
+    }
+
+
+}

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+
+export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
+role: string;
+});

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+
+export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
+user_uid: string;
+});