Server autocloses on crash

Thumbnails are now retrieved using file UID

.dockerignore

@@ -1,7 +0,0 @@
-.git
-db
-appdata
-audio
-video
-subscriptions
-users

.eslintrc.json

@@ -1,20 +0,0 @@
-{
-    "env": {
-        "browser": true,
-        "es2021": true
-    },
-    "extends": [
-        "eslint:recommended",
-        "plugin:@typescript-eslint/recommended"
-    ],
-    "parser": "@typescript-eslint/parser",
-    "parserOptions": {
-        "ecmaVersion": 12,
-        "sourceType": "module"
-    },
-    "plugins": [
-        "@typescript-eslint"
-    ],
-    "rules": {
-    }
-}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,12 +27,5 @@ If applicable, add screenshots to help explain your problem.
 - YoutubeDL-Material version
 - Docker tag: <tag> (optional)
 
-Ideally you'd copy the info as presented on the "About" dialogue
-in YoutubeDL-Material.
-(for that, click on the three dots on the top right and then
-check "installation details". On later versions of YoutubeDL-
-Material you will find pretty much all the crucial information
-here that we need in most cases!)
-
 **Additional context**
-Add any other context about the problem here. For example, a YouTube link.
+Add any other context about the problem here.

.github/workflows/build.yml

@@ -25,21 +25,8 @@ jobs:
           cd backend
           npm install
           sudo npm install -g @angular/cli
-      - name: Set hash
-        id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: Get current date
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
-      - name: create-json
-        id: create-json
-        uses: jsdaniell/create-json@1.1.2
-        with:
-          name: "version.json"
-          json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
-          dir: 'backend/'
       - name: build
-        run: npm run build
+        run: ng build --prod
       - name: prepare artifact upload
         shell: pwsh
         run: |
@@ -51,7 +38,7 @@ jobs:
           Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
-          New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
+          New-Item -Path ./build/youtubedl-material -Name users
           Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
       - name: upload build artifact

.github/workflows/docker-pr.yml

@@ -1,27 +0,0 @@
-name: docker-pr
-
-on:
-  pull_request:
-    branches: [master]
-
-jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout code
-        uses: actions/checkout@v2
-      - name: Set hash
-        id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: Get current date
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
-      - name: create-json
-        id: create-json
-        uses: jsdaniell/create-json@1.1.2
-        with:
-          name: "version.json"
-          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
-          dir: 'backend/'
-      - name: Build docker images
-        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker-release.yml

@@ -13,19 +13,6 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
-      - name: Set hash
-        id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: Get current date
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
-      - name: create-json
-        id: create-json
-        uses: jsdaniell/create-json@1.1.2
-        with:
-          name: "version.json"
-          json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
-          dir: 'backend/'
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
       - name: setup multi-arch docker build

.github/workflows/docker.yml

@@ -10,19 +10,6 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
-      - name: Set hash
-        id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: Get current date
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
-      - name: create-json
-        id: create-json
-        uses: jsdaniell/create-json@1.1.2
-        with:
-          name: "version.json"
-          json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
-          dir: 'backend/'
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
       - name: setup multi-arch docker build
@@ -39,8 +26,4 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          # Defaults:
-          #     DOCKERHUB_USERNAME  : tzahi12345
-          #     DOCKERHUB_REPO      : youtubedl-material
-          #     DOCKERHUB_MASTER_TAG: nightly
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}:${{secrets.DOCKERHUB_MASTER_TAG}}
+          tags: tzahi12345/youtubedl-material:nightly

.gitignore

@@ -25,7 +25,6 @@
 !.vscode/extensions.json
 
 # misc
-/.angular/cache
 /.sass-cache
 /connect.lock
 /coverage
@@ -66,13 +65,4 @@ backend/appdata/logs/error.log
 backend/appdata/users.json
 backend/users/*
 backend/appdata/cookies.txt
-backend/public
-src/assets/i18n/*.json
-
-# User Files
-db/
-appdata/
-audio/
-video/
-subscriptions/
-users/
+backend/public

Dockerfile

@@ -1,25 +1,9 @@
-FROM ubuntu:20.04 AS ffmpeg
+FROM alpine:3.12 as frontend
 
-ENV DEBIAN_FRONTEND=noninteractive
+RUN apk add --no-cache \
+  npm
 
-COPY docker-build.sh .
-RUN sh ./docker-build.sh
-
-FROM ubuntu:20.04 as frontend
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install \
-  curl \
-  gnupg && \
-  curl -sL https://deb.nodesource.com/setup_12.x  | bash - && \
-  apt-get -y install \
-  nodejs \
-  # YARN: brings along npm, solves dependency conflicts,
-  # spares us this spaghetti approach: https://stackoverflow.com/a/60547197
-  yarn && \
-  apt-get install -f && \
-  npm config set strict-ssl false && \
-  npm install -g @angular/cli
+RUN npm install -g @angular/cli
 
 WORKDIR /build
 COPY [ "package.json", "package-lock.json", "/build/" ]
@@ -27,45 +11,38 @@ RUN npm install
 
 COPY [ "angular.json", "tsconfig.json", "/build/" ]
 COPY [ "src/", "/build/src/" ]
-RUN npm run build
+RUN ng build --prod
 
 #--------------#
 
-FROM ubuntu:20.04
+FROM alpine:3.12
 
 ENV UID=1000 \
   GID=1000 \
-  USER=youtube \
-  NO_UPDATE_NOTIFIER=true
+  USER=youtube
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV NO_UPDATE_NOTIFIER=true
+ENV FOREVER_ROOT=/app/.forever
 
-RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
+RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
 
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get update && apt-get -y install \
+RUN apk add --no-cache \
+  ffmpeg \
   npm \
   python2 \
   python3 \
-  atomicparsley && \
-  apt-get install -f && \
-  apt-get autoremove --purge && \
-  apt-get autoremove && \
-  apt-get clean && \
-  rm -rf /var/lib/apt
+  su-exec \
+  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
+    atomicparsley
 
 WORKDIR /app
-COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
-COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
-ENV PM2_HOME=/app/pm2
-RUN npm config set strict-ssl false && \
-  npm install pm2 -g && \
-  npm install && chown -R $UID:$GID ./
+RUN npm install forever -g
+RUN npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
-# ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime", "pm2.config.js" ]
+ENTRYPOINT [ "/app/entrypoint.sh" ]
+CMD [ "forever", "app.js" ]

README.md

@@ -6,22 +6,10 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 
-<hr>
-
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
 Check out the prerequisites, and go to the installation section. Easy as pie!
@@ -79,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
 
 To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
 
-Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
+Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
 
 The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
 
@@ -89,10 +77,6 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ## Docker
 
-### Host-specific instructions
-
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
-
 ### Setup
 
 If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
@@ -122,12 +106,6 @@ To get started, go to the settings menu and enable the public API from the *Extr
 
 Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above.
 
-## iOS Shortcut 
-
-If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?)
-
-You can download Shortcut [here.](https://routinehub.co/shortcut/10283/)
-
 ## Contributing
 
 If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away.
@@ -146,16 +124,12 @@ Official translators:
 * German - UnlimitedCookies
 * Chinese - TyRoyal
 
-See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
+See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
 
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details
 
-## Legal Disclaimer
-
-This project is in no way affiliated with Google LLC, Alphabet Inc. or YouTube (or their subsidiaries) nor endorsed by them.
-
 ## Acknowledgments
 
 * youtube-dl

SECURITY.md

@@ -1,21 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-Currently all work on this project goes into the nightly builds.
-4.2's RELEASE build is now quite old and should be considered legacy.
-We urge users to use the nightly releases, because the project
-constantly sees fixes.
-
-| Version       | Supported          |
-| ------------- | ------------------ |
-| 4.2 Nightlies | :white_check_mark: |
-| 4.2 Release   | :x:                |
-| < 4.2         | :x:                |
-
-## Reporting a Vulnerability
-
-Please file an issue in our GitHub's repo, because this app
-isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
-
-We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

angular.json

@@ -17,6 +17,7 @@
         "build": {
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
+            "aot": true,
             "outputPath": "backend/public",
             "index": "src/index.html",
             "main": "src/main.ts",
@@ -32,17 +33,7 @@
             "styles": [
               "src/styles.scss"
             ],
-            "scripts": [],
-            "vendorChunk": true,
-            "extractLicenses": false,
-            "buildOptimizer": false,
-            "sourceMap": true,
-            "optimization": false,
-            "namedChunks": true,
-            "allowedCommonJsDependencies": [
-              "rxjs",
-              "crypto-js"
-            ]
+            "scripts": []
           },
           "configurations": {
             "production": {
@@ -55,6 +46,7 @@
               "optimization": true,
               "outputHashing": "all",
               "namedChunks": false,
+              "aot": true,
               "extractLicenses": true,
               "vendorChunk": false,
               "buildOptimizer": true,
@@ -68,8 +60,7 @@
             "es": {
               "localize": ["es"]
             }
-          },
-          "defaultConfiguration": ""
+          }
         },
         "serve": {
           "builder": "@angular-devkit/build-angular:dev-server",
@@ -161,6 +152,16 @@
               "src/backend"
             ]
           }
+        },
+        "lint": {
+          "builder": "@angular-devkit/build-angular:tslint",
+          "options": {
+            "tsConfig": [
+              "src/tsconfig.app.json",
+              "src/tsconfig.spec.json"
+            ],
+            "exclude": []
+          }
         }
       }
     },
@@ -175,6 +176,15 @@
             "protractorConfig": "./protractor.conf.js",
             "devServerTarget": "youtube-dl-material:serve"
           }
+        },
+        "lint": {
+          "builder": "@angular-devkit/build-angular:tslint",
+          "options": {
+            "tsConfig": [
+              "e2e/tsconfig.e2e.json"
+            ],
+            "exclude": []
+          }
         }
       }
     }

armhf.Dockerfile

@@ -0,0 +1,49 @@
+FROM alpine:3.12 as frontend
+
+RUN apk add --no-cache \
+  npm \
+  curl
+
+RUN npm install -g @angular/cli
+
+WORKDIR /build
+
+RUN curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
+
+COPY [ "package.json", "package-lock.json", "/build/" ]
+RUN npm install
+
+COPY [ "angular.json", "tsconfig.json", "/build/" ]
+COPY [ "src/", "/build/src/" ]
+RUN ng build --prod
+
+#--------------#
+
+FROM arm32v7/alpine:3.12
+
+COPY --from=frontend /build/qemu-arm-static /usr/bin
+
+ENV UID=1000 \
+  GID=1000 \
+  USER=youtube
+
+RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
+
+RUN apk add --no-cache \
+  ffmpeg \
+  npm \
+  python2 \
+  su-exec \
+  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
+    atomicparsley
+
+WORKDIR /app
+COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
+RUN npm install && chown -R $UID:$GID ./
+
+COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
+COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
+
+EXPOSE 17442
+ENTRYPOINT [ "/app/entrypoint.sh" ]
+CMD [ "node", "app.js" ]

backend/.eslintrc.json

@@ -1,18 +0,0 @@
-{
-    "env": {
-        "node": true,
-        "es2021": true
-    },
-    "extends": [
-        "eslint:recommended"
-    ],
-    "parser": "esprima",
-    "parserOptions": {
-        "ecmaVersion": 12,
-        "sourceType": "module"
-    },
-    "plugins": [],
-    "rules": {
-    },
-    "root": true
-}

backend/appdata/default.json

@@ -12,16 +12,14 @@
       "custom_args": "",
       "safe_download_override": false,
       "include_thumbnail": true,
-      "include_metadata": true,
-      "max_concurrent_downloads": 5,
-      "download_rate_limit": ""
+      "include_metadata": true
     },
     "Extra": {
       "title_top": "YoutubeDL-Material",
       "file_manager_enabled": true,
       "allow_quality_select": true,
       "download_only_mode": false,
-      "allow_autoplay": true,
+      "allow_multi_download_mode": true,
       "enable_downloads_manager": true,
       "allow_playlist_categorization": true
     },
@@ -32,9 +30,7 @@
       "youtube_API_key": "",
       "use_twitch_API": false,
       "twitch_API_key": "",
-      "twitch_auto_download_chat": false,
-      "use_sponsorblock_API": false,
-      "generate_NFO_files": false
+      "twitch_auto_download_chat": false
     },
     "Themes": {
       "default_theme": "default",
@@ -59,7 +55,7 @@
       }
     },
     "Database": {
-      "use_local_db": true,
+      "use_local_db": false,
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
@@ -69,8 +65,8 @@
       "multi_user_mode": false,
       "allow_advanced_download": false,
       "use_cookies": false,
-      "jwt_expiration": 86400,
+      "jwt_expiration": 86400, 
       "logger_level": "info"
     }
   }
-}
+}

backend/authentication/auth.js

@@ -1,8 +1,7 @@
+const path = require('path');
 const config_api = require('../config');
 const consts = require('../consts');
-const logger = require('../logger');
-const db_api = require('../db');
-
+const fs = require('fs-extra');
 const jwt = require('jsonwebtoken');
 const { uuid } = require('uuidv4');
 const bcrypt = require('bcryptjs');
@@ -13,24 +12,20 @@ var JwtStrategy = require('passport-jwt').Strategy,
     ExtractJwt = require('passport-jwt').ExtractJwt;
 
 // other required vars
+let logger = null;
+let db_api = null;
 let SERVER_SECRET = null;
 let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function () {
+exports.initialize = function(db_api, input_logger) {
+  setLogger(input_logger)
+  setDB(db_api);
+
   /*************************
    * Authentication module
    ************************/
-
-  if (db_api.database_initialized) {
-    setupRoles();
-  } else {
-      db_api.database_initialized_bs.subscribe(init => {
-          if (init) setupRoles();
-      });
-  }
-
   saltRounds = 10;
 
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -58,39 +53,12 @@ exports.initialize = function () {
   }));
 }
 
-const setupRoles = async () => {
-  const required_roles = {
-    admin: {
-        permissions: [
-            'filemanager',
-            'settings',
-            'subscriptions',
-            'sharing',
-            'advanced_download',
-            'downloads_manager'
-        ]
-    },
-    user: {
-        permissions: [
-            'filemanager',
-            'subscriptions',
-            'sharing'
-        ]
-    }
-  }
+function setLogger(input_logger) {
+  logger = input_logger;
+}
 
-  const role_keys = Object.keys(required_roles);
-  for (let i = 0; i < role_keys.length; i++) {
-    const role_key = role_keys[i];
-    const role_in_db = await db_api.getRecord('roles', {key: role_key});
-    if (!role_in_db) {
-      // insert task metadata into table if missing
-      await db_api.insertRecordIntoTable('roles', {
-          key: role_key,
-          permissions: required_roles[role_key]['permissions']
-      });
-    }
-  }
+function setDB(input_db_api) {
+  db_api = input_db_api;
 }
 
 exports.passport = require('passport');
@@ -172,7 +140,7 @@ exports.registerUser = async function(req, res) {
 
 exports.login = async (username, password) => {
   const user = await db_api.getRecord('users', {name: username});
-  if (!user) { logger.error(`User ${username} not found`); return false }
+  if (!user) { logger.error(`User ${username} not found`); false }
   if (user.auth_method && user.auth_method !== 'internal') { return false }
   return await bcrypt.compare(password, user.passhash) ? user : false;
 }
@@ -323,12 +291,17 @@ exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false
   return file;
 }
 
+exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
+  users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
+  return true;
+}
+
 exports.removePlaylist = async function(user_uid, playlistID) {
   await db_api.removeRecord('playlist', {playlistID: playlistID});
   return true;
 }
 
-exports.getUserPlaylists = async function(user_uid) {
+exports.getUserPlaylists = async function(user_uid, user_files = null) {
   return await db_api.getRecords('playlists', {user_uid: user_uid});
 }
 

backend/categories.js

@@ -1,6 +1,19 @@
+const config_api = require('./config');
 const utils = require('./utils');
-const logger = require('./logger');
-const db_api = require('./db');
+
+var logger = null;
+var db = null;
+var users_db = null;
+var db_api = null;
+
+function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
+function setLogger(input_logger) { logger = input_logger; }
+
+function initialize(input_db, input_users_db, input_logger, input_db_api) {
+    setDB(input_db, input_users_db, input_db_api);
+    setLogger(input_logger);
+}
+
 /*
 
 Categories:
@@ -59,7 +72,7 @@ async function getCategoriesAsPlaylists(files = null) {
     const categories_as_playlists = [];
     const available_categories = await getCategories();
     if (available_categories && files) {
-        for (let category of available_categories) {
+        for (category of available_categories) {
             const files_that_match = utils.addUIDsToCategory(category, files);
             if (files_that_match && files_that_match.length > 0) {
                 category['thumbnailURL'] = files_that_match[0].thumbnailURL;
@@ -112,23 +125,24 @@ function applyCategoryRules(file_json, rules, category_name) {
     return rules_apply;
 }
 
-// async function addTagToVideo(tag, video, user_uid) {
-//     // TODO: Implement
-// }
+async function addTagToVideo(tag, video, user_uid) {
+    // TODO: Implement
+}
 
-// async function removeTagFromVideo(tag, video, user_uid) {
-//     // TODO: Implement
-// }
+async function removeTagFromVideo(tag, video, user_uid) {
+    // TODO: Implement
+}
 
-// // adds tag to list of existing tags (used for tag suggestions)
-// async function addTagToExistingTags(tag) {
-//     const existing_tags = db.get('tags').value();
-//     if (!existing_tags.includes(tag)) {
-//         db.get('tags').push(tag).write();
-//     }
-// }
+// adds tag to list of existing tags (used for tag suggestions)
+async function addTagToExistingTags(tag) {
+    const existing_tags = db.get('tags').value();
+    if (!existing_tags.includes(tag)) {
+        db.get('tags').push(tag).write();
+    }
+}
 
 module.exports = {
+    initialize: initialize,
     categorize: categorize,
     getCategories: getCategories,
     getCategoriesAsPlaylists: getCategoriesAsPlaylists

backend/config.js

@@ -1,5 +1,3 @@
-const logger = require('./logger');
-
 const fs = require('fs');
 
 let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
@@ -7,7 +5,11 @@ const debugMode = process.env.YTDL_MODE === 'debug';
 
 let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
 
-function initialize() {
+var logger = null;
+function setLogger(input_logger) { logger = input_logger; }
+
+function initialize(input_logger) {
+    setLogger(input_logger);
     ensureConfigFileExists();
     ensureConfigItemsExist();
 }
@@ -95,13 +97,13 @@ function getConfigItem(key) {
     }
     let path = CONFIG_ITEMS[key]['path'];
     const val = Object.byString(config_json, path);
-    if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
+    if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
         logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
         setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
         return Object.byString(DEFAULT_CONFIG, path);
     }
     return Object.byString(config_json, path);
-}
+};
 
 function setConfigItem(key, value) {
     let success = false;
@@ -173,7 +175,7 @@ module.exports = {
     globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
 }
 
-const DEFAULT_CONFIG = {
+DEFAULT_CONFIG = {
     "YoutubeDLMaterial": {
       "Host": {
         "url": "http://example.com",
@@ -187,16 +189,14 @@ const DEFAULT_CONFIG = {
         "custom_args": "",
         "safe_download_override": false,
         "include_thumbnail": true,
-        "include_metadata": true,
-        "max_concurrent_downloads": 5,
-        "download_rate_limit": ""
+        "include_metadata": true
       },
       "Extra": {
         "title_top": "YoutubeDL-Material",
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_autoplay": true,
+        "allow_multi_download_mode": true,
         "enable_downloads_manager": true,
         "allow_playlist_categorization": true
       },
@@ -207,9 +207,7 @@ const DEFAULT_CONFIG = {
         "youtube_API_key": "",
         "use_twitch_API": false,
         "twitch_API_key": "",
-        "twitch_auto_download_chat": false,
-        "use_sponsorblock_API": false,
-        "generate_NFO_files": false
+        "twitch_auto_download_chat": false
       },
       "Themes": {
         "default_theme": "default",
@@ -218,7 +216,7 @@ const DEFAULT_CONFIG = {
       "Subscriptions": {
         "allow_subscriptions": true,
         "subscriptions_base_path": "subscriptions/",
-        "subscriptions_check_interval": "86400",
+        "subscriptions_check_interval": "300",
         "redownload_fresh_uploads": false
       },
       "Users": {
@@ -234,7 +232,7 @@ const DEFAULT_CONFIG = {
         }
       },
       "Database": {
-        "use_local_db": true,
+        "use_local_db": false,
         "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
       },
       "Advanced": {

backend/consts.js

@@ -1,4 +1,4 @@
-exports.CONFIG_ITEMS = {
+let CONFIG_ITEMS = {
     // Host
     'ytdl_url': {
         'key': 'ytdl_url',
@@ -42,14 +42,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_include_metadata',
         'path': 'YoutubeDLMaterial.Downloader.include_metadata'
     },
-    'ytdl_max_concurrent_downloads': {
-        'key': 'ytdl_max_concurrent_downloads',
-        'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
-    },
-    'ytdl_download_rate_limit': {
-        'key': 'ytdl_download_rate_limit',
-        'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
-    },
 
     // Extra
     'ytdl_title_top': {
@@ -68,9 +60,9 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_autoplay': {
-        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
+    'ytdl_allow_multi_download_mode': {
+        'key': 'ytdl_allow_multi_download_mode',
+        'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -110,15 +102,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
     },
-    'ytdl_use_sponsorblock_api': {
-        'key': 'ytdl_use_sponsorblock_api',
-        'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
-    },
-    'ytdl_generate_nfo_files': {
-        'key': 'ytdl_generate_nfo_files',
-        'path': 'YoutubeDLMaterial.API.generate_NFO_files'
-    },
-
 
     // Themes
     'ytdl_default_theme': {
@@ -143,6 +126,10 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_subscriptions_check_interval',
         'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
     },
+    'ytdl_subscriptions_check_interval': {
+        'key': 'ytdl_subscriptions_check_interval',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
+    },
     'ytdl_subscriptions_redownload_fresh_uploads': {
         'key': 'ytdl_subscriptions_redownload_fresh_uploads',
         'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
@@ -211,7 +198,7 @@ exports.CONFIG_ITEMS = {
     }
 };
 
-exports.AVAILABLE_PERMISSIONS = [
+AVAILABLE_PERMISSIONS = [
     'filemanager',
     'settings',
     'subscriptions',
@@ -220,6 +207,8 @@ exports.AVAILABLE_PERMISSIONS = [
     'downloads_manager'
 ];
 
-exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
-
-exports.CURRENT_VERSION = 'v4.2';
+module.exports = {
+    CONFIG_ITEMS: CONFIG_ITEMS,
+    AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS,
+    CURRENT_VERSION: 'v4.2'
+}

backend/db.js

@@ -1,31 +1,24 @@
 var fs = require('fs-extra')
 var path = require('path')
-const { MongoClient } = require("mongodb");
-const { uuid } = require('uuidv4');
-
-const config_api = require('./config');
 var utils = require('./utils')
-const logger = require('./logger');
+const { uuid } = require('uuidv4');
+const config_api = require('./config');
+const { MongoClient } = require("mongodb");
 
 const low = require('lowdb')
 const FileSync = require('lowdb/adapters/FileSync');
-const { BehaviorSubject } = require('rxjs');
 const local_adapter = new FileSync('./appdata/local_db.json');
 const local_db = low(local_adapter);
 
-let database = null;
-exports.database_initialized = false;
-exports.database_initialized_bs = new BehaviorSubject(false);
+var logger = null;
+var db = null;
+var users_db = null;
+var database = null;
 
 const tables = {
     files: {
         name: 'files',
-        primary_key: 'uid',
-        text_search: {
-            title: 'text',
-            uploader: 'text',
-            uid: 'text'
-        }
+        primary_key: 'uid'
     },
     playlists: {
         name: 'playlists',
@@ -50,14 +43,6 @@ const tables = {
         name: 'roles',
         primary_key: 'key'
     },
-    download_queue: {
-        name: 'download_queue',
-        primary_key: 'uid'
-    },
-    tasks: {
-        name: 'tasks',
-        primary_key: 'key'
-    },
     test: {
         name: 'test'
     }
@@ -77,16 +62,20 @@ function setDB(input_db, input_users_db) {
     exports.users_db = input_users_db
 }
 
-exports.initialize = (input_db, input_users_db) => {
+function setLogger(input_logger) {
+    logger = input_logger;
+}
+
+exports.initialize = (input_db, input_users_db, input_logger) => {
     setDB(input_db, input_users_db);
+    setLogger(input_logger);
 
     // must be done here to prevent getConfigItem from being called before init
     using_local_db = config_api.getConfigItem('ytdl_use_local_db');
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
-    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
-    if (using_local_db && !custom_connection_string) return;
+    if (using_local_db) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;
 
@@ -142,13 +131,8 @@ exports._connectToDB = async (custom_connection_string = null) => {
 
         tables_list.forEach(async table => {
             const primary_key = tables[table]['primary_key'];
-            if (primary_key) {
-                await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
-            }
-            const text_search = tables[table]['text_search'];
-            if (text_search) {
-                await database.collection(table).createIndex(text_search);
-            }
+            if (!primary_key) return;
+            await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
         });
         return true;
     } catch(err) {
@@ -160,17 +144,51 @@ exports._connectToDB = async (custom_connection_string = null) => {
     }
 }
 
-exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
-    if (!file_object) file_object = generateFileObject(file_path, type);
+exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => {
+    let db_path = null;
+    const file_id = utils.removeFileExtension(file_path);
+    if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
+    if (!file_object) {
+        logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
+        return false;
+    }
+
+    utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path);
+
+    // add thumbnail path
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path);
+
+    // if category exists, only include essential info
+    if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
+
+    // modify duration
+    if (cropFileSettings) {
+        file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
+    }
+
+    if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
+
+    const file_obj = await registerFileDBManual(file_object);
+
+    // remove metadata JSON if needed
+    if (!config_api.getConfigItem('ytdl_include_metadata')) {
+        utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
+    }
+
+    return file_obj;
+}
+
+exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
+    if (!file_object) file_object = generateFileObject2(file_path, type);
     if (!file_object) {
         logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
         return false;
     }
 
-    utils.fixVideoMetadataPerms(file_path, type);
+    utils.fixVideoMetadataPerms2(file_path, type);
 
     // add thumbnail path
-    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type);
 
     // if category exists, only include essential info
     if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
@@ -187,7 +205,7 @@ exports.registerFileDB = async (file_path, type, user_uid = null, category = nul
 
     // remove metadata JSON if needed
     if (!config_api.getConfigItem('ytdl_include_metadata')) {
-        utils.deleteJSONFile(file_path, type)
+        utils.deleteJSONFile2(file_path, type)
     }
 
     return file_obj;
@@ -205,12 +223,38 @@ async function registerFileDBManual(file_object) {
     return file_object;
 }
 
-function generateFileObject(file_path, type) {
-    var jsonobj = utils.getJSON(file_path, type);
+function generateFileObject(id, type, customPath = null, sub = null) {
+    if (!customPath && sub) {
+        customPath = getAppendedBasePathSub(sub, config_api.getConfigItem('ytdl_subscriptions_base_path'));
+    }
+    var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
     if (!jsonobj) {
         return null;
-    } else if (!jsonobj['_filename']) {
-        logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
+    }
+    const ext = (type === 'audio') ? '.mp3' : '.mp4'
+    const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
+    // console.
+    var stats = fs.statSync(path.join(file_path));
+
+    var title = jsonobj.title;
+    var url = jsonobj.webpage_url;
+    var uploader = jsonobj.uploader;
+    var upload_date = jsonobj.upload_date;
+    upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
+
+    var size = stats.size;
+
+    var thumbnail = jsonobj.thumbnail;
+    var duration = jsonobj.duration;
+    var isaudio = type === 'audio';
+    var description = jsonobj.description;
+    var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
+    return file_obj;
+}
+
+function generateFileObject2(file_path, type) {
+    var jsonobj = utils.getJSON(file_path, type);
+    if (!jsonobj) {
         return null;
     }
     const ext = (type === 'audio') ? '.mp3' : '.mp4'
@@ -222,7 +266,8 @@ function generateFileObject(file_path, type) {
     var title = jsonobj.title;
     var url = jsonobj.webpage_url;
     var uploader = jsonobj.uploader;
-    var upload_date = utils.formatDateString(jsonobj.upload_date);
+    var upload_date = jsonobj.upload_date;
+    upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
 
     var size = stats.size;
 
@@ -305,7 +350,6 @@ exports.getFileDirectoriesAndDBs = async () => {
 }
 
 exports.importUnregisteredFiles = async () => {
-    const imported_files = [];
     const dirs_to_check = await exports.getFileDirectoriesAndDBs();
 
     // run through check list and check each file to see if it's missing from the db
@@ -318,21 +362,33 @@ exports.importUnregisteredFiles = async () => {
             const file = files[j];
 
             // check if file exists in db, if not add it
-            const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
-            const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
+            const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id}))
             if (!file_is_registered) {
                 // add additional info
-                const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
-                if (file_obj) {
-                    imported_files.push(file_obj['uid']);
-                    logger.verbose(`Added discovered file to the database: ${file.id}`);
-                } else {
-                    logger.error(`Failed to import ${file['path']} automatically.`);
-                }
+                await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
+                logger.verbose(`Added discovered file to the database: ${file.id}`);
             }
         }
     }
-    return imported_files;
+
+}
+
+exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
+    const preimported_file_paths = [];
+
+    const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        // check if file exists in db, if not add it
+        const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
+        if (!file_is_registered) {
+            // add additional info
+            await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
+            preimported_file_paths.push(file['path']);
+            logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
+        }
+    }
+    return preimported_file_paths;
 }
 
 exports.addMetadataPropertyToDB = async (property_key) => {
@@ -463,8 +519,8 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
     var thumbnailPath = `${filePathNoExtension}.webp`;
     var altThumbnailPath = `${filePathNoExtension}.jpg`;
 
-    jsonPath = path.join(__dirname, jsonPath);
-    altJSONPath = path.join(__dirname, altJSONPath);
+    jsonPath = path.join(jsonPath);
+    altJSONPath = path.join(altJSONPath);
 
     let jsonExists = await fs.pathExists(jsonPath);
     let thumbnailExists = await fs.pathExists(thumbnailPath);
@@ -564,22 +620,7 @@ exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
         return true;
     }
 
-    if (replaceFilter) {
-        const output = await database.collection(table).bulkWrite([
-            {
-                deleteMany: {
-                    filter: replaceFilter
-                }
-            },
-            {
-                insertOne: {
-                    document: doc
-                }
-            }
-        ]);
-        logger.debug(`Inserted doc into ${table} with filter: ${JSON.stringify(replaceFilter)}`);
-        return !!(output['result']['ok']);
-    }
+    if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
 
     const output = await database.collection(table).insertOne(doc);
     logger.debug(`Inserted doc into ${table}`);
@@ -636,28 +677,13 @@ exports.getRecord = async (table, filter_obj) => {
     return await database.collection(table).findOne(filter_obj);
 }
 
-exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
+exports.getRecords = async (table, filter_obj = null) => {
     // local db override
     if (using_local_db) {
-        let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
-        if (sort) {
-            cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
-        }
-        if (range) {
-            cursor = cursor.slice(range[0], range[1]);
-        }
-        return !return_count ? cursor : cursor.length;
+        return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
     }
 
-    const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
-    if (sort) {
-        cursor.sort({[sort['by']]: sort['order']});
-    }
-    if (range) {
-        cursor.skip(range[0]).limit(range[1] - range[0]);
-    }
-
-    return !return_count ? await cursor.toArray() : await cursor.count();
+    return filter_obj ? await database.collection(table).find(filter_obj).toArray()  : await database.collection(table).find().toArray();
 }
 
 // Update
@@ -755,86 +781,26 @@ exports.removeRecord = async (table, filter_obj) => {
     return !!(output['result']['ok']);
 }
 
-// exports.removeRecordsByUIDBulk = async (table, uids) => {
-//     // local db override
-//     if (using_local_db) {
-//         applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
-//         return true;
-//     }
-
-//     const table_collection = database.collection(table);
-        
-//     let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
-
-//     const item_ids_to_remove = 
-
-//     for (let i = 0; i < item_ids_to_update.length; i++) {
-//         const item_id_to_update = item_ids_to_update[i];
-//         bulk.find({[key_label]: item_id_to_update }).updateOne({
-//             "$set": update_obj[item_id_to_update]
-//         });
-//     }
-
-//     const output = await bulk.execute();
-//     return !!(output['result']['ok']);
-// }
-
-
-exports.findDuplicatesByKey = async (table, key) => {
-    let duplicates = [];
-    if (using_local_db) {
-        // this can probably be optimized
-        const all_records = await exports.getRecords(table);
-        const existing_records = {};
-        for (let i = 0; i < all_records.length; i++) {
-            const record = all_records[i];
-            const value = record[key];
-
-            if (existing_records[value]) {
-                duplicates.push(record);
-            }
-
-            existing_records[value] = true;
-        }
-        return duplicates;
-    }
-    
-    const duplicated_values = await database.collection(table).aggregate([
-        {"$group" : { "_id": `$${key}`, "count": { "$sum": 1 } } },
-        {"$match": {"_id" :{ "$ne" : null } , "count" : {"$gt": 1} } }, 
-        {"$project": {[key] : "$_id", "_id" : 0} }
-    ]).toArray();
-
-    for (let i = 0; i < duplicated_values.length; i++) {
-        const duplicated_value = duplicated_values[i];
-        const duplicated_records = await exports.getRecords(table, duplicated_value, false);
-        if (duplicated_records.length > 1) {
-            duplicates = duplicates.concat(duplicated_records.slice(1, duplicated_records.length));
-        }
-    }
-    return duplicates;
-}
-
-exports.removeAllRecords = async (table = null, filter_obj = null) => {
+exports.removeAllRecords = async (table = null) => {
     // local db override
     const tables_to_remove = table ? [table] : tables_list;
-    logger.debug(`Removing all records from: ${tables_to_remove} with filter: ${JSON.stringify(filter_obj)}`)
     if (using_local_db) {
+        logger.debug(`Removing all records from: ${tables_to_remove}`)
         for (let i = 0; i < tables_to_remove.length; i++) {
             const table_to_remove = tables_to_remove[i];
-            if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
-            else local_db.assign({[table_to_remove]: []}).write();
-            logger.debug(`Successfully removed records from ${table_to_remove}`);
+            local_db.assign({[table_to_remove]: []}).write();
+            logger.debug(`Removed all records from ${table_to_remove}`);
         }
         return true;
     }
 
     let success = true;
+    logger.debug(`Removing all records from: ${tables_to_remove}`)
     for (let i = 0; i < tables_to_remove.length; i++) {
         const table_to_remove = tables_to_remove[i];
 
-        const output = await database.collection(table_to_remove).deleteMany(filter_obj ? filter_obj : {});
-        logger.debug(`Successfully removed records from ${table_to_remove}`);
+        const output = await database.collection(table_to_remove).deleteMany({});
+        logger.debug(`Removed all records from ${table_to_remove}`);
         success &= !!(output['result']['ok']);
     }
     return success;
@@ -988,52 +954,6 @@ const createDownloadsRecords = (downloads) => {
     return new_downloads;
 }
 
-exports.backupDB = async () => {
-    const backup_dir = path.join('appdata', 'db_backup');
-    fs.ensureDirSync(backup_dir);
-    const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
-    const path_to_backups = path.join(backup_dir, backup_file_name);
-
-    logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
-
-    const table_to_records = {};
-    for (let i = 0; i < tables_list.length; i++) {
-        const table = tables_list[i];
-        table_to_records[table] = await exports.getRecords(table);
-    }
-
-    fs.writeJsonSync(path_to_backups, table_to_records);
-
-    return backup_file_name;
-}
-
-exports.restoreDB = async (file_name) => {
-    const path_to_backup = path.join('appdata', 'db_backup', file_name);
-
-    logger.debug('Reading database backup file.');
-    const table_to_records = fs.readJSONSync(path_to_backup);
-
-    if (!table_to_records) {
-        logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`);
-        return false;
-    }
-
-    logger.debug('Clearing database.');
-    await exports.removeAllRecords();
-
-    logger.debug('Database cleared! Beginning restore.');
-    let success = true;
-    for (let i = 0; i < tables_list.length; i++) {
-        const table = tables_list[i];
-        if (!table_to_records[table] || table_to_records[table].length === 0) continue;
-        success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
-    }
-
-    logger.debug('Restore finished!');
-
-    return success;
-}
-
 exports.transferDB = async (local_to_remote) => {
     const table_to_records = {};
     for (let i = 0; i < tables_list.length; i++) {
@@ -1043,8 +963,9 @@ exports.transferDB = async (local_to_remote) => {
 
     using_local_db = !local_to_remote;
     if (local_to_remote) {
-        logger.debug('Backup up DB...');
-        await exports.backupDB();
+        // backup local DB
+        logger.debug('Backup up Local DB...');
+        await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
         const db_connected = await exports.connectToDB(5, true);
         if (!db_connected) {
             logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1067,8 +988,6 @@ exports.transferDB = async (local_to_remote) => {
 
     config_api.setConfigItem('ytdl_use_local_db', using_local_db);
 
-    logger.debug('Transfer finished!');
-
     return success;
 }
 
@@ -1088,28 +1007,10 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
             if (filter_prop_value === undefined || filter_prop_value === null) {
                 filtered &= record[filter_prop] === undefined || record[filter_prop] === null
             } else {
-                if (typeof filter_prop_value === 'object') {
-                    if (filter_prop_value['$regex']) {
-                        filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
-                    }
-                } else {
-                    filtered &= record[filter_prop] === filter_prop_value;
-                }
+                filtered &= record[filter_prop] === filter_prop_value;
             }
         }
         return filtered;
     });
     return return_val;
-}
-
-// archive helper functions
-
-async function writeToBlacklist(type, line) {
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}
+}

backend/downloader.js

@@ -1,630 +0,0 @@
-const fs = require('fs-extra');
-const { uuid } = require('uuidv4');
-const path = require('path');
-const mergeFiles = require('merge-files');
-const NodeID3 = require('node-id3')
-const Mutex = require('async-mutex').Mutex;
-
-const youtubedl = require('youtube-dl');
-
-const logger = require('./logger');
-const config_api = require('./config');
-const twitch_api = require('./twitch');
-const { create } = require('xmlbuilder2');
-const categories_api = require('./categories');
-const utils = require('./utils');
-const db_api = require('./db');
-
-const mutex = new Mutex();
-let should_check_downloads = true;
-
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
-if (db_api.database_initialized) {
-    setupDownloads();
-} else {
-    db_api.database_initialized_bs.subscribe(init => {
-        if (init) setupDownloads();
-    });
-}
-
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
-    return await mutex.runExclusive(async () => {
-        const download = {
-            url: url,
-            type: type,
-            title: '',
-            user_uid: user_uid,
-            sub_id: sub_id,
-            sub_name: sub_name,
-            options: options,
-            uid: uuid(),
-            step_index: 0,
-            paused: false,
-            running: false,
-            finished_step: true,
-            error: null,
-            percent_complete: null,
-            finished: false,
-            timestamp_start: Date.now()
-        };
-        await db_api.insertRecordIntoTable('download_queue', download);
-    
-        should_check_downloads = true;
-        return download;
-    });
-}
-
-exports.pauseDownload = async (download_uid) => {
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (download['paused']) {
-        logger.warn(`Download ${download_uid} is already paused!`);
-        return false;
-    } else if (download['finished']) {
-        logger.info(`Download ${download_uid} could not be paused before completing.`);
-        return false;
-    }
-
-    return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
-}
-
-exports.resumeDownload = async (download_uid) => {
-    return await mutex.runExclusive(async () => {
-        const download = await db_api.getRecord('download_queue', {uid: download_uid});
-        if (!download['paused']) {
-            logger.warn(`Download ${download_uid} is not paused!`);
-            return false;
-        }
-
-        const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
-        should_check_downloads = true;
-        return success;
-    })
-}
-
-exports.restartDownload = async (download_uid) => {
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    await exports.clearDownload(download_uid);
-    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
-    
-    should_check_downloads = true;
-    return success;
-}
-
-exports.cancelDownload = async (download_uid) => {
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (download['cancelled']) {
-        logger.warn(`Download ${download_uid} is already cancelled!`);
-        return false;
-    } else if (download['finished']) {
-        logger.info(`Download ${download_uid} could not be cancelled before completing.`);
-        return false;
-    }
-    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
-}
-
-exports.clearDownload = async (download_uid) => {
-    return await db_api.removeRecord('download_queue', {uid: download_uid});
-}
-
-async function handleDownloadError(download_uid, error_message) {
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
-}
-
-async function setupDownloads() {
-    await fixDownloadState();
-    setInterval(checkDownloads, 1000);
-}
-
-async function fixDownloadState() {
-    const downloads = await db_api.getRecords('download_queue');
-    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
-    const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
-    for (let i = 0; i < running_downloads.length; i++) {
-        const running_download = running_downloads[i];
-        const update_obj = {finished_step: true, paused: true, running: false};
-        if (running_download['step_index'] > 0) {
-            update_obj['step_index'] = running_download['step_index'] - 1;
-        }
-        await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
-    }
-}
-
-async function checkDownloads() {
-    if (!should_check_downloads) return;
-
-    const downloads = await db_api.getRecords('download_queue');
-    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
-
-    await mutex.runExclusive(async () => {
-        // avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
-        const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
-        if (running_downloads.length === 0) {
-            should_check_downloads = false;
-            logger.verbose('Disabling checking downloads as none are available.');
-        }
-        return;
-    });
-
-    let running_downloads_count = downloads.filter(download => download['running']).length;
-    const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
-    for (let i = 0; i < waiting_downloads.length; i++) {
-        const waiting_download = waiting_downloads[i];
-        const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
-        if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
-
-        if (waiting_download['finished_step'] && !waiting_download['finished']) {
-            // move to next step
-            running_downloads_count++;
-            if (waiting_download['step_index'] === 0) {
-                collectInfo(waiting_download['uid']);
-            } else if (waiting_download['step_index'] === 1) {
-                downloadQueuedFile(waiting_download['uid']);
-            }
-        }
-    }
-}
-
-async function collectInfo(download_uid) {
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (download['paused']) {
-        return;
-    }
-    logger.verbose(`Collecting info for download ${download_uid}`);
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
-
-    const url = download['url'];
-    const type = download['type'];
-    const options = download['options'];
-
-    if (download['user_uid'] && !options.customFileFolderPath) {
-        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const user_path = path.join(usersFileFolder, download['user_uid'], type);
-        options.customFileFolderPath = user_path + path.sep;
-    }
-
-    let args = await exports.generateArgs(url, type, options, download['user_uid']);
-
-    // get video info prior to download
-    let info = await getVideoInfoByURL(url, args, download_uid);
-
-    if (!info) {
-        // info failed, error presumably already recorded
-        return;
-    }
-
-    let category = null;
-
-    // check if it fits into a category. If so, then get info again using new args
-    if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
-
-    // set custom output if the category has one and re-retrieve info so the download manager has the right file name
-    if (category && category['custom_output']) {
-        options.customOutput = category['custom_output'];
-        options.noRelativePath = true;
-        args = await exports.generateArgs(url, type, options, download['user_uid']);
-        info = await getVideoInfoByURL(url, args, download_uid);
-    }
-
-    // setup info required to calculate download progress
-
-    const expected_file_size = utils.getExpectedFileSize(info);
-
-    const files_to_check_for_progress = [];
-
-    // store info in download for future use
-    if (Array.isArray(info)) {
-        for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
-    } else {
-        files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
-    }
-
-    const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
-    await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
-                                                                    finished_step: true,
-                                                                    running: false,
-                                                                    options: options,
-                                                                    files_to_check_for_progress: files_to_check_for_progress,
-                                                                    expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
-                                                                });
-}
-
-async function downloadQueuedFile(download_uid) {
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    if (download['paused']) {
-        return;
-    }
-    logger.verbose(`Downloading ${download_uid}`);
-    return new Promise(async resolve => {
-        const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
-        const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
-        await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
-
-        const url = download['url'];
-        const type = download['type'];
-        const options = download['options'];
-        const args = download['args'];
-        const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
-        if (options.customFileFolderPath) {
-            fileFolderPath = options.customFileFolderPath;
-        }
-        fs.ensureDirSync(fileFolderPath);
-
-        const start_time = Date.now();
-
-        const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
-
-        // download file
-        youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
-            const file_objs = [];
-            let end_time = Date.now();
-            let difference = (end_time - start_time)/1000;
-            logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
-            clearInterval(download_checker);
-            if (err) {
-                logger.error(err.stderr);
-                await handleDownloadError(download_uid, err.stderr);
-                resolve(false);
-                return;
-            } else if (output) {
-                if (output.length === 0 || output[0].length === 0) {
-                    // ERROR!
-                    const error_message = `No output received for video download, check if it exists in your archive.`;
-                    await handleDownloadError(download_uid, error_message);
-                    logger.warn(error_message);
-                    resolve(false);
-                    return;
-                }
-
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                    } catch(e) {
-                        output_json = null;
-                    }
-
-                    if (!output_json) {
-                        continue;
-                    }
-
-                    // get filepath with no extension
-                    const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
-
-                    const ext = type === 'audio' ? '.mp3' : '.mp4';
-                    var full_file_path = filepath_no_extension + ext;
-                    var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
-
-                    if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
-                            let vodId = url.split('twitch.tv/videos/')[1];
-                            vodId = vodId.split('?')[0];
-                            twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
-                    }
-
-                    // renames file if necessary due to bug
-                    if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
-                        try {
-                            fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
-                            logger.info('Renamed ' + file_name + '.webm to ' + file_name);
-                        } catch(e) {
-                            logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
-                        }
-                    }
-
-                    if (type === 'audio') {
-                        let tags = {
-                            title: output_json['title'],
-                            artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
-                        }
-                        let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
-                        if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
-                    }
-
-                    if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
-                        exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
-                    }
-
-                    if (options.cropFileSettings) {
-                        await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
-                    }
-
-                    // registers file in DB
-                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
-
-                    file_objs.push(file_obj);
-                }
-
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
-                }
-
-                let container = null;
-
-                if (file_objs.length > 1) {
-                    // create playlist
-                    const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
-                } else if (file_objs.length === 1) {
-                    container = file_objs[0];
-                } else {
-                    const error_message = 'Downloaded file failed to result in metadata object.';
-                    logger.error(error_message);
-                    await handleDownloadError(download_uid, error_message);
-                }
-
-                const file_uids = file_objs.map(file_obj => file_obj.uid);
-                await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
-                resolve();
-            }
-        });
-    });
-}
-
-// helper functions
-
-exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
-    const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
-    const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
-
-    const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
-    const globalArgs = config_api.getConfigItem('ytdl_custom_args');
-    const useCookies = config_api.getConfigItem('ytdl_use_cookies');
-    const is_audio = type === 'audio';
-
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
-
-    if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
-
-    const customArgs = options.customArgs;
-    let customOutput = options.customOutput;
-    const customQualityConfiguration = options.customQualityConfiguration;
-
-    // video-specific args
-    const selectedHeight = options.selectedHeight;
-
-    // audio-specific args
-    const maxBitrate = options.maxBitrate;
-
-    const youtubeUsername = options.youtubeUsername;
-    const youtubePassword = options.youtubePassword;
-
-    let downloadConfig = null;
-    let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
-    const is_youtube = url.includes('youtu');
-    if (!is_audio && !is_youtube) {
-        // tiktok videos fail when using the default format
-        qualityPath = null;
-    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
-        qualityPath = ['-f', 'bestvideo+bestaudio']
-    }
-
-    if (customArgs) {
-        downloadConfig = customArgs.split(',,');
-    } else {
-        if (customQualityConfiguration) {
-            qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
-        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
-        } else if (is_audio) {
-            qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
-        }
-
-        if (customOutput) {
-            customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
-            downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
-        } else {
-            downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
-        }
-
-        if (qualityPath) downloadConfig.push(...qualityPath);
-
-        if (is_audio && !options.skip_audio_args) {
-            downloadConfig.push('-x');
-            downloadConfig.push('--audio-format', 'mp3');
-        }
-
-        if (youtubeUsername && youtubePassword) {
-            downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
-        }
-
-        if (useCookies) {
-            if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
-                downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
-            } else {
-                logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
-            }
-        }
-
-        const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
-        const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
-        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
-            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
-        }
-
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
-        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
-            downloadConfig.push('--write-thumbnail');
-        }
-
-        if (globalArgs && globalArgs !== '') {
-            // adds global args
-            if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
-                // if global args has an output, replce the original output with that of global args
-                const original_output_index = downloadConfig.indexOf('-o');
-                downloadConfig.splice(original_output_index, 2);
-            }
-            downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
-        }
-
-        if (options.additionalArgs && options.additionalArgs !== '') {
-            downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
-        }
-
-        const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
-        if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
-            downloadConfig.push('-r', rate_limit);
-        }
-        
-        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
-        if (default_downloader === 'yt-dlp') {
-            downloadConfig.push('--no-clean-infojson');
-        }
-
-    }
-
-    // filter out incompatible args
-    downloadConfig = filterArgs(downloadConfig, is_audio);
-
-    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
-    return downloadConfig;
-}
-
-async function getVideoInfoByURL(url, args = [], download_uid = null) {
-    return new Promise(resolve => {
-        // remove bad args
-        const new_args = [...args];
-
-        const archiveArgIndex = new_args.indexOf('--download-archive');
-        if (archiveArgIndex !== -1) {
-            new_args.splice(archiveArgIndex, 2);
-        }
-
-        new_args.push('--dump-json');
-
-        youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
-            if (output) {
-                let outputs = [];
-                try {
-                    for (let i = 0; i < output.length; i++) {
-                        let output_json = null;
-                        try {
-                            output_json = JSON.parse(output[i]);
-                        } catch(e) {
-                            output_json = null;
-                        }
-    
-                        if (!output_json) {
-                            continue;
-                        }
-
-                        outputs.push(output_json);
-                    }
-                    resolve(outputs.length === 1 ? outputs[0] : outputs);
-                } catch(e) {
-                    const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
-                    logger.error(error);
-                    if (download_uid) {
-                        await handleDownloadError(download_uid, error);
-                    }
-                    resolve(null);
-                }
-            } else {
-                let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
-                if (err.stderr) error_message += `\n\n${err.stderr}`;
-                logger.error(error_message);
-                if (download_uid) {
-                    await handleDownloadError(download_uid, error_message);
-                }
-                resolve(null);
-            }
-        });
-    });
-}
-
-function filterArgs(args, isAudio) {
-    const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
-    const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
-    return args.filter(x => !args_to_remove.includes(x));
-}
-
-async function checkDownloadPercent(download_uid) {
-    /*
-    This is more of an art than a science, we're just selecting files that start with the file name,
-    thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
-
-    Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
-    be divided by the "total expected bytes."
-    */
-
-    const download = await db_api.getRecord('download_queue', {uid: download_uid});
-    const files_to_check_for_progress = download['files_to_check_for_progress'];
-    const resulting_file_size = download['expected_file_size'];
-
-    if (!resulting_file_size) return;
-
-    let sum_size = 0;
-    for (let i = 0; i < files_to_check_for_progress.length; i++) {
-        const file_to_check_for_progress = files_to_check_for_progress[i];
-        const dir = path.dirname(file_to_check_for_progress);
-        if (!fs.existsSync(dir)) continue;
-        fs.readdir(dir, async (err, files) => {
-            for (let j = 0; j < files.length; j++) {
-                const file = files[j];
-                if (!file.includes(path.basename(file_to_check_for_progress))) continue;
-                try {
-                    const file_stats = fs.statSync(path.join(dir, file));
-                    if (file_stats && file_stats.size) {
-                        sum_size += file_stats.size;
-                    }
-                } catch (e) {}
-            }
-            
-            const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
-            await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
-        });
-    }
-}
-
-exports.generateNFOFile = (info, output_path) => {
-    const nfo_obj = {
-        episodedetails: {
-            title: info['fulltitle'],
-            episode: info['playlist_index'] ? info['playlist_index'] : undefined,
-            premiered: utils.formatDateString(info['upload_date']),
-            plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
-            director: info['artist'] ? info['artist'] : info['uploader']
-        }
-    };
-    const doc = create(nfo_obj);
-    const xml = doc.end({ prettyPrint: true });
-    fs.writeFileSync(output_path, xml);
-}
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join(archivePath);
-    }
-}

backend/ecosystem.config.js

@@ -1,8 +0,0 @@
-module.exports = {
-  apps : [{
-    name   : "YoutubeDL-Material",
-    script : "./app.js",
-    watch  : "placeholder",
-    watch_delay: 5000
-  }]
-}

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="pm2-runtime pm2.config.js"
+CMD="forever app.js"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then

backend/logger.js

@@ -1,23 +0,0 @@
-const winston = require('winston');
-
-let debugMode = process.env.YTDL_MODE === 'debug';
-
-const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
-    return `${timestamp} ${level.toUpperCase()}: ${message}`;
-});
-const logger = winston.createLogger({
-    level: 'info',
-    format: winston.format.combine(winston.format.timestamp(), defaultFormat),
-    defaultMeta: {},
-    transports: [
-      //
-      // - Write to all logs with level `info` and below to `combined.log`
-      // - Write all logs error (and below) to `error.log`.
-      //
-      new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
-      new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
-      new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
-    ]
-});
-
-module.exports = logger;

backend/main.js

@@ -1,6 +1,7 @@
 const { app, BrowserWindow } = require('electron');
 const path = require('path');
 const url = require('url');
+const server = require('./app');
 
 let win;
 
@@ -8,13 +9,7 @@ function createWindow() {
   win = new BrowserWindow({ width: 800, height: 600 });
 
   // load the dist folder from Angular
-  win.loadURL(
-    url.format({
-      pathname: path.join(__dirname, `/dist/index.html`),
-      protocol: 'file:',
-      slashes: true
-    })
-  );
+  win.loadURL('http://localhost:17442')  //ADD THIS
 
   // The following is optional and will open the DevTools:
   // win.webContents.openDevTools()

backend/package.json

@@ -2,11 +2,14 @@
   "name": "backend",
   "version": "1.0.0",
   "description": "backend for YoutubeDL-Material",
-  "main": "index.js",
+  "main": "main.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "nodemon app.js",
-    "debug": "set YTDL_MODE=debug && node app.js"
+    "debug": "set YTDL_MODE=debug && node app.js",
+    "electron": "electron main.js",
+    "pack": "electron-builder --dir",
+    "dist": "electron-builder"
   },
   "nodemonConfig": {
     "ignore": [
@@ -19,6 +22,13 @@
       "restart_general.json"
     ]
   },
+  "build": {
+    "appId": "youtubedl.material",
+    "mac": {
+      "category": "public.app-category.utilities"
+    },
+    "files": ["!audio/*", "!video/*", "!users/*", "!subscriptions/*", "!appdata/*"]
+  },
   "repository": {
     "type": "git",
     "url": ""
@@ -30,42 +40,44 @@
   },
   "homepage": "",
   "dependencies": {
-    "archiver": "^5.3.1",
-    "async": "^3.2.3",
-    "async-mutex": "^0.3.1",
-    "axios": "^0.21.2",
+    "archiver": "^3.1.1",
+    "async": "^3.1.0",
+    "axios": "^0.21.1",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
-    "express": "^4.17.3",
+    "exe": "^1.0.2",
+    "express": "^4.17.1",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "glob": "^7.1.6",
     "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
     "merge-files": "^0.1.2",
-    "mocha": "^9.2.2",
-    "moment": "^2.29.2",
+    "mocha": "^8.4.0",
+    "moment": "^2.29.1",
     "mongodb": "^3.6.9",
     "multer": "^1.4.2",
-    "node-fetch": "^2.6.7",
+    "node-fetch": "^2.6.1",
     "node-id3": "^0.1.14",
-    "node-schedule": "^2.1.0",
     "nodemon": "^2.0.7",
     "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.0",
-    "passport-ldapauth": "^3.0.1",
+    "passport-ldapauth": "^2.1.4",
     "passport-local": "^1.0.0",
     "progress": "^2.0.3",
     "ps-node": "^0.1.6",
     "read-last-lines": "^1.7.2",
-    "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
     "uuidv4": "^6.0.6",
-    "winston": "^3.7.2",
-    "xmlbuilder2": "^3.0.2",
+    "winston": "^3.2.1",
     "youtube-dl": "^3.0.2"
+  },
+  "devDependencies": {
+    "electron": "^13.1.7",
+    "electron-builder": "^22.11.7"
   }
 }

backend/pm2.config.js

@@ -1,9 +0,0 @@
-module.exports = {
-    apps : [{
-      name   : "YoutubeDL-Material",
-      script : "./app.js",
-      watch  : "placeholder",
-      out_file: "/dev/null",
-      error_file: "/dev/null"
-    }]
-}

backend/subscriptions.js

@@ -1,15 +1,28 @@
-const fs = require('fs-extra');
-const path = require('path');
-const youtubedl = require('youtube-dl');
+const FileSync = require('lowdb/adapters/FileSync')
 
+var fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+var path = require('path');
+
+var youtubedl = require('youtube-dl');
 const config_api = require('./config');
-const utils = require('./utils');
-const logger = require('./logger');
+const twitch_api = require('./twitch');
+var utils = require('./utils');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
-const db_api = require('./db');
-const downloader_api = require('./downloader');
+var logger = null;
+var db = null;
+var users_db = null;
+let db_api = null;
+
+function setDB(input_db_api) { db_api = input_db_api }
+function setLogger(input_logger) { logger = input_logger; }
+
+function initialize(input_db_api, input_logger) {
+    setDB(input_db_api);
+    setLogger(input_logger);
+}
 
 async function subscribe(sub, user_uid = null) {
     const result_obj = {
@@ -33,13 +46,13 @@ async function subscribe(sub, user_uid = null) {
         sub['user_uid'] = user_uid ? user_uid : undefined;
         await db_api.insertRecordIntoTable('subscriptions', sub);
 
-        let success = await getSubscriptionInfo(sub);
+        let success = await getSubscriptionInfo(sub, user_uid);
 
         if (success) {
             getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
-        }
+        };
 
         result_obj.success = success;
         result_obj.sub = sub;
@@ -48,12 +61,18 @@ async function subscribe(sub, user_uid = null) {
 
 }
 
-async function getSubscriptionInfo(sub) {
+async function getSubscriptionInfo(sub, user_uid = null) {
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
     // get videos
     let downloadConfig = ['--dump-json', '--playlist-end', '1'];
     let useCookies = config_api.getConfigItem('ytdl_use_cookies');
     if (useCookies) {
-        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+        if (await fs.pathExists(path.join('appdata', 'cookies.txt'))) {
             downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
         } else {
             logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -95,6 +114,22 @@ async function getSubscriptionInfo(sub) {
                         }
                     }
 
+                    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useArchive && !sub.archive) {
+                        // must create the archive
+                        const archive_dir = path.join(basePath, 'archives', sub.name);
+                        const archive_path = path.join(archive_dir, 'archive.txt');
+
+                        // creates archive directory and text file if it doesn't exist
+                        fs.ensureDirSync(archive_dir);
+                        fs.ensureFileSync(archive_path);
+
+                        // updates subscription
+                        sub.archive = archive_dir;
+
+                        await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
+                    }
+
                     // TODO: get even more info
 
                     resolve(true);
@@ -111,23 +146,9 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    let result_obj = { success: false, error: '' };
 
     let id = sub.id;
-
-    const sub_files = await db_api.getRecords('files', {sub_id: id});
-    for (let i = 0; i < sub_files.length; i++) {
-        const sub_file = sub_files[i];
-        if (config_api.descriptors[sub_file['uid']]) {
-            try {
-                for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
-                    config_api.descriptors[sub_file['uid']][i].destroy();
-                }
-            } catch(e) {
-                continue;
-            }
-        }
-    }
-
     await db_api.removeRecord('subscriptions', {id: id});
     await db_api.removeAllRecords('files', {sub_id: id});
 
@@ -141,7 +162,6 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         if (sub.archive && (await fs.pathExists(sub.archive))) {
             const archive_file_path = path.join(sub.archive, 'archive.txt');
             // deletes archive if it exists
-            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
             if (await fs.pathExists(archive_file_path)) {
                 await fs.unlink(archive_file_path);
             }
@@ -165,10 +185,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
 
     let filePath = appendedBasePath;
     const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
-    var jsonPath = path.join(__dirname,filePath,name+'.info.json');
-    var videoFilePath = path.join(__dirname,filePath,name+ext);
-    var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
-    var altImageFilePath = path.join(__dirname,filePath,name+'.webp');
+    var jsonPath = path.join(filePath,name+'.info.json');
+    var videoFilePath = path.join(filePath,name+ext);
+    var imageFilePath = path.join(filePath,name+'.jpg');
+    var altImageFilePath = path.join(filePath,name+'.webp');
 
     const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
         fs.pathExists(jsonPath),
@@ -229,15 +249,30 @@ async function getVideosForSub(sub, user_uid = null) {
     let appendedBasePath = getAppendedBasePath(sub, basePath);
     fs.ensureDirSync(appendedBasePath);
 
+    let multiUserMode = null;
+    if (user_uid) {
+        multiUserMode = {
+            user: user_uid,
+            file_path: appendedBasePath
+        }
+    }
+
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose('Subscription: getting videos for subscription ' + sub.name);
 
     return new Promise(async resolve => {
+        const preimported_file_paths = [];
+        const PREIMPORT_INTERVAL = 5000;
+        const preregister_check = setInterval(async () => {
+            if (sub.streamingOnly) return;
+            await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
+        }, PREIMPORT_INTERVAL);
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+            clearInterval(preregister_check);
 
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
@@ -245,21 +280,19 @@ async function getVideosForSub(sub, user_uid = null) {
                 if (err.stderr.includes('This video is unavailable')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        // TODO: reimplement
-
-                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
-                        // for (let i = 0; i < outputs.length; i++) {
-                        //     const output = JSON.parse(outputs[i]);
-                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                        //         // we found a video that errored! add it to the archive to prevent future errors
-                        //         if (sub.archive) {
-                        //             archive_dir = sub.archive;
-                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                        //             fs.appendFileSync(archive_path, output['id']);
-                        //         }
-                        //     }
-                        // }
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        for (let i = 0; i < outputs.length; i++) {
+                            const output = JSON.parse(outputs[i]);
+                            await handleOutputJSON(sub, output, i === 0, multiUserMode)
+                            if (err.stderr.includes(output['id']) && archive_path) {
+                                // we found a video that errored! add it to the archive to prevent future errors
+                                if (sub.archive) {
+                                    archive_dir = sub.archive;
+                                    archive_path = path.join(archive_dir, 'archive.txt')
+                                    fs.appendFileSync(archive_path, output['id']);
+                                }
+                            }
+                        }
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
@@ -267,68 +300,41 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
                 resolve(false);
             } else if (output) {
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                     logger.verbose('No additional videos to download for ' + sub.name);
                     resolve(true);
                     return;
                 }
-
-                const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
                         output_json = JSON.parse(output[i]);
-                        output_jsons.push(output_json);
                     } catch(e) {
                         output_json = null;
                     }
                     if (!output_json) {
                         continue;
                     }
+
+                    const reset_videos = i === 0;
+                    await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
                 }
 
-                const files_to_download = await getFilesToDownload(sub, output_jsons);
-                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
-
-                for (let j = 0; j < files_to_download.length; j++) {
-                    const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
+                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
                 }
 
-                resolve(files_to_download);
-           }
+                resolve(true);
+            }
         });
     }, err => {
         logger.error(err);
         updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+        clearInterval(preregister_check);
     });
 }
 
-function generateOptionsForSubscriptionDownload(sub, user_uid) {
-    let basePath = null;
-    if (user_uid)
-        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
-    else
-        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
-
-    let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
-
-    const base_download_options = {
-        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
-        customFileFolderPath: getAppendedBasePath(sub, basePath),
-        customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
-        additionalArgs: sub.custom_args
-    }
-
-    return base_download_options;
-}
-
 async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
     // get basePath
     let basePath = null;
@@ -343,14 +349,14 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
 
-    let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
+    let fullOutput = `${appendedBasePath}/${file_output}.%(ext)s`;
     if (desired_path) {
-        fullOutput = `"${desired_path}.%(ext)s"`;
+        fullOutput = `${desired_path}.%(ext)s`;
     } else if (sub.custom_output) {
-        fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
+        fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
     }
 
-    let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
+    let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
 
     let qualityPath = null;
     if (sub.type && sub.type === 'audio') {
@@ -365,7 +371,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     downloadConfig.push(...qualityPath)
 
     if (sub.custom_args) {
-        const customArgsArray = sub.custom_args.split(',,');
+        customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
             // if custom args has a custom quality, replce the original quality with that of custom args
             const original_output_index = downloadConfig.indexOf('-f');
@@ -380,11 +386,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     if (useArchive && !redownload) {
         if (sub.archive) {
             archive_dir = sub.archive;
-            if (sub.type && sub.type === 'audio') {
-                archive_path = path.join(archive_dir, 'merged_audio.txt');
-            } else {
-                archive_path = path.join(archive_dir, 'merged_video.txt');
-            }            
+            archive_path = path.join(archive_dir, 'archive.txt')
         }
         downloadConfig.push('--download-archive', archive_path);
     }
@@ -400,7 +402,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     let useCookies = config_api.getConfigItem('ytdl_use_cookies');
     if (useCookies) {
-        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+        if (await fs.pathExists(path.join('appdata', 'cookies.txt'))) {
             downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
         } else {
             logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -411,37 +413,46 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push('--write-thumbnail');
     }
 
-    const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
-    if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
-        downloadConfig.push('-r', rate_limit);
-    }
-
-    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
-    if (default_downloader === 'yt-dlp') {
-        downloadConfig.push('--no-clean-infojson');
-    }
-
     return downloadConfig;
 }
 
-async function getFilesToDownload(sub, output_jsons) {
-    const files_to_download = [];
-    for (let i = 0; i < output_jsons.length; i++) {
-        const output_json = output_jsons[i];
-        const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
-        if (file_missing) {
-            const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
-            if (file_with_path_exists) {
-                // or maybe just overwrite???
-                logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
-            }
-            files_to_download.push(output_json);
+async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
+    // TODO: remove streaming only mode
+    if (false && sub.streamingOnly) {
+        if (reset_videos) {
+            sub_db.assign({videos: []}).write();
+        }
+
+        // remove unnecessary info
+        output_json.formats = null;
+
+        // add to db
+        sub_db.get('videos').push(output_json).write();
+    } else {
+        path_object = path.parse(output_json['_filename']);
+        const path_string = path.format(path_object);
+
+        const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id});
+        if (file_exists) {
+            // TODO: fix issue where files of different paths due to custom path get downloaded multiple times
+            // file already exists in DB, return early to avoid reseting the download date
+            return;
+        }
+
+        await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);
+
+        const url = output_json['webpage_url'];
+        if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
+            && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                const file_name = path.basename(output_json['_filename']);
+                const id = file_name.substring(0, file_name.length-4);
+                let vodId = url.split('twitch.tv/videos/')[1];
+                vodId = vodId.split('?')[0];
+                twitch_api.downloadTwitchChatByVODID(vodId, id, sub.type, multiUserMode.user, sub);
         }
     }
-    return files_to_download;
 }
 
-
 async function getSubscriptions(user_uid = null) {
     return await db_api.getRecords('subscriptions', {user_uid: user_uid});
 }
@@ -449,7 +460,7 @@ async function getSubscriptions(user_uid = null) {
 async function getAllSubscriptions() {
     const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
-    return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
+    return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
 }
 
 async function getSubscription(subID) {
@@ -460,7 +471,7 @@ async function getSubscriptionByName(subName, user_uid = null) {
     return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
 }
 
-async function updateSubscription(sub) {
+async function updateSubscription(sub, user_uid = null) {
     await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
     return true;
 }
@@ -471,30 +482,28 @@ async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     });
 }
 
-async function updateSubscriptionProperty(sub, assignment_obj) {
+async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
     // TODO: combine with updateSubscription
     await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
     return true;
 }
 
-async function setFreshUploads(sub) {
-    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+async function setFreshUploads(sub, user_uid) {
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub_files.forEach(async file => {
-        if (current_date === file['upload_date'].replace(/-/g, '')) {
+    sub.videos.forEach(async video => {
+        if (current_date === video['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const file_uid = file['uid'];
-            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
+            const video_uid = video['uid'];
+            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
-    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub_files.forEach(async file => {
-        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(file, sub, user_uid)
+    sub.videos.forEach(async video => {
+        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
+            await checkVideoIfBetterExists(video, sub, user_uid)
         }
     });
 }
@@ -516,18 +525,19 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
 }
 
 // helper functions
 
 function getAppendedBasePath(sub, base_path) {
+
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
@@ -541,6 +551,7 @@ module.exports = {
     unsubscribe            : unsubscribe,
     deleteSubscriptionFile : deleteSubscriptionFile,
     getVideosForSub        : getVideosForSub,
-    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
-    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
+    setLogger              : setLogger,
+    initialize             : initialize,
+    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple
 }

backend/tasks.js

@@ -1,195 +0,0 @@
-const db_api = require('./db');
-const youtubedl_api = require('./youtube-dl');
-
-const fs = require('fs-extra');
-const logger = require('./logger');
-const scheduler = require('node-schedule');
-
-const TASKS = {
-    backup_local_db: {
-        run: db_api.backupDB,
-        title: 'Backup DB',
-        job: null
-    },
-    missing_files_check: {
-        run: checkForMissingFiles,
-        confirm: deleteMissingFiles,
-        title: 'Missing files check',
-        job: null
-    },
-    missing_db_records: {
-        run: db_api.importUnregisteredFiles,
-        title: 'Import missing DB records',
-        job: null
-    },
-    duplicate_files_check: {
-        run: checkForDuplicateFiles,
-        confirm: removeDuplicates,
-        title: 'Find duplicate files in DB',
-        job: null
-    },
-    youtubedl_update_check: {
-        run: youtubedl_api.checkForYoutubeDLUpdate,
-        confirm: youtubedl_api.updateYoutubeDL,
-        title: 'Update youtube-dl',
-        job: null
-    }
-}
-
-function scheduleJob(task_key, schedule) {
-    // schedule has to be converted from our format to one node-schedule can consume
-    let converted_schedule = null;
-    if (schedule['type'] === 'timestamp') {
-        converted_schedule = new Date(schedule['data']['timestamp']);
-    } else if (schedule['type'] === 'recurring') {
-        const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
-        const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
-        const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
-        converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
-    } else {
-        logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
-        return null;
-    }
-
-    return scheduler.scheduleJob(converted_schedule, async () => {
-        const task_state = await db_api.getRecord('tasks', {key: task_key});
-        if (task_state['running'] || task_state['confirming']) {
-            logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
-            return;
-        }
-        
-        // remove schedule if it's a one-time task
-        if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
-        // we're just "running" the task, any confirmation should be user-initiated
-        exports.executeRun(task_key);
-    });
-}
-
-if (db_api.database_initialized) {
-    exports.setupTasks();
-} else {
-    db_api.database_initialized_bs.subscribe(init => {
-        if (init) exports.setupTasks();
-    });
-}
-
-exports.setupTasks = async () => {
-    const tasks_keys = Object.keys(TASKS);
-    for (let i = 0; i < tasks_keys.length; i++) {
-        const task_key = tasks_keys[i];
-        const task_in_db = await db_api.getRecord('tasks', {key: task_key});
-        if (!task_in_db) {
-            // insert task metadata into table if missing
-            await db_api.insertRecordIntoTable('tasks', {
-                key: task_key,
-                title: TASKS[task_key]['title'],
-                last_ran: null,
-                last_confirmed: null,
-                running: false,
-                confirming: false,
-                data: null,
-                error: null,
-                schedule: null,
-                options: {}
-            });
-        } else {
-            // reset task if necessary
-            await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
-
-            // schedule task and save job
-            if (task_in_db['schedule']) {
-                // prevent timestamp schedules from being set to the past
-                if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) {
-                    await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
-                    continue;
-                }
-                TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
-            }
-        }
-    }
-}
-
-exports.executeTask = async (task_key) => {
-    if (!TASKS[task_key]) {
-        logger.error(`Task ${task_key} does not exist!`);
-        return;
-    }
-    logger.verbose(`Executing task ${task_key}`);
-    await exports.executeRun(task_key);
-    if (!TASKS[task_key]['confirm']) return;
-    await exports.executeConfirm(task_key);
-    logger.verbose(`Finished executing ${task_key}`);
-}
-
-exports.executeRun = async (task_key) => {
-    logger.verbose(`Running task ${task_key}`);
-    // don't set running to true when backup up DB as it will be stick "running" if restored
-    if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
-    const data = await TASKS[task_key].run();
-    await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
-    logger.verbose(`Finished running task ${task_key}`);
-}
-
-exports.executeConfirm = async (task_key) => {
-    logger.verbose(`Confirming task ${task_key}`);
-    if (!TASKS[task_key]['confirm']) {
-        return null;
-    }
-    await db_api.updateRecord('tasks', {key: task_key}, {confirming: true});
-    const task_obj = await db_api.getRecord('tasks', {key: task_key});
-    const data = task_obj['data'];
-    await TASKS[task_key].confirm(data);
-    await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
-    logger.verbose(`Finished confirming task ${task_key}`);
-}
-
-exports.updateTaskSchedule = async (task_key, schedule) => {
-    logger.verbose(`Updating schedule for task ${task_key}`);
-    await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
-    if (TASKS[task_key]['job']) {
-        TASKS[task_key]['job'].cancel();
-    }
-    if (schedule) {
-        TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
-    }
-}
-
-// missing files check
-
-async function checkForMissingFiles() {
-    const missing_files = [];
-    const all_files = await db_api.getRecords('files');
-    for (let i = 0; i < all_files.length; i++) {
-        const file_to_check = all_files[i];
-        const file_exists = fs.existsSync(file_to_check['path']);
-        if (!file_exists) missing_files.push(file_to_check['uid']);
-    }
-    return {uids: missing_files};
-}
-
-async function deleteMissingFiles(data) {
-    const uids = data['uids'];
-    for (let i = 0; i < uids.length; i++) {
-        const uid = uids[i];
-        await db_api.removeRecord('files', {uid: uid});
-    }
-}
-
-// duplicate files check
-
-async function checkForDuplicateFiles() {
-    const duplicate_files = await db_api.findDuplicatesByKey('files', 'path');
-    const duplicate_uids = duplicate_files.map(duplicate_file => duplicate_file['uid']);
-    if (duplicate_uids && duplicate_uids.length > 0) {
-        return {uids: duplicate_uids};
-    }
-    return {uids: []};
-}
-
-async function removeDuplicates(data) {
-    for (let i = 0; i < data['uids'].length; i++) {
-        await db_api.removeRecord('files', {uid: data['uids'][i]});
-    }
-}
-
-exports.TASKS = TASKS;

backend/test/tests.js

@@ -40,7 +40,7 @@ const subscriptions_api = require('../subscriptions');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
 
-db_api.initialize(db, users_db);
+db_api.initialize(db, users_db, logger);
 
 
 describe('Database', async function() {
@@ -70,17 +70,6 @@ describe('Database', async function() {
             const success = await db_api.getRecord('test', {test: 'test'});
             assert(success);
         });
-
-        it('Restore db', async function() {
-            const db_stats = await db_api.getDBStats();
-            
-            const file_name = await db_api.backupDB();
-            await db_api.restoreDB(file_name);
-
-            const new_db_stats = await db_api.getDBStats();
-
-            assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats));
-        });
     });
 
     describe('Export', function() {
@@ -94,37 +83,12 @@ describe('Database', async function() {
             await db_api.removeAllRecords('test');
         });
         it('Add and read record', async function() {
-            this.timeout(120000);
             await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
             const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
             assert(added_record['test_add'] === 'test');
             await db_api.removeRecord('test', {test_add: 'test'});
         });
 
-        it('Find duplicates by key', async function() {
-            const test_duplicates = [
-                {
-                    test: 'testing',
-                    key: '1'
-                },
-                {
-                    test: 'testing',
-                    key: '2'
-                },
-                {
-                    test: 'testing_missing',
-                    key: '3'
-                },
-                {
-                    test: 'testing',
-                    key: '4'
-                }
-            ];
-            await db_api.insertRecordsIntoTable('test', test_duplicates);
-            const duplicates = await db_api.findDuplicatesByKey('test', 'test');
-            console.log(duplicates);
-        });
-
         it('Update record', async function() {
             await db_api.insertRecordIntoTable('test', {test_update: 'test'});
             await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
@@ -158,7 +122,6 @@ describe('Database', async function() {
         });
 
         it('Bulk add', async function() {
-            this.timeout(120000);
             const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
             const test_records = [];
             for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
@@ -323,159 +286,5 @@ describe('Multi User', async function() {
     //         assert(video_obj);
     //     });
     // });
-
-});
     
-describe('Downloader', function() {
-    const downloader_api = require('../downloader');
-    const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
-    const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
-    const options = {
-        ui_uid: uuid(),
-        user: 'admin'
-    }
-
-    beforeEach(async function() {
-        await db_api.connectToDB();
-        await db_api.removeAllRecords('download_queue');
-    });
-
-    it('Get file info', async function() {
-
-    });
-
-    it('Download file', async function() {
-        this.timeout(300000); 
-        const returned_download = await downloader_api.createDownload(url, 'video', options);
-        console.log(returned_download);
-        await utils.wait(20000);
-
-    });
-
-    it('Queue file', async function() {
-        this.timeout(300000); 
-        const returned_download = await downloader_api.createDownload(url, 'video', options);
-        console.log(returned_download);
-        await utils.wait(20000);
-    });
-
-    it('Pause file', async function() {
-
-    });
-
-    it('Generate args', async function() {
-        const args = await downloader_api.generateArgs(url, 'video', options);
-        console.log(args);
-    });
-
-    it('Generate args - subscription', async function() {
-        subscriptions_api.initialize(db_api, logger);
-        const sub = await subscriptions_api.getSubscription(sub_id);
-        const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
-        const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
-        console.log(args);
-    });
-
-    it('Generate kodi NFO file', async function() {
-        const nfo_file_path = './test/sample.nfo';
-        if (fs.existsSync(nfo_file_path)) {
-            fs.unlinkSync(nfo_file_path);
-        }
-        const sample_json = fs.readJSONSync('./test/sample.info.json');
-        downloader_api.generateNFOFile(sample_json, nfo_file_path);
-        assert(fs.existsSync(nfo_file_path), true);
-        fs.unlinkSync(nfo_file_path);
-    });
-});
-
-describe('Tasks', function() {
-    const tasks_api = require('../tasks');
-    beforeEach(async function() {
-        await db_api.connectToDB();
-        await db_api.removeAllRecords('tasks');
-
-        const dummy_task = {
-            run: async () => { await utils.wait(500); return true; },
-            confirm: async () => { await utils.wait(500); return true; },
-            title: 'Dummy task',
-            job: null
-        };
-        tasks_api.TASKS['dummy_task'] = dummy_task;
-
-        await tasks_api.initialize();
-    });
-    it('Backup db', async function() {
-        const backups_original = await utils.recFindByExt('appdata', 'bak');
-        const original_length = backups_original.length;
-        await tasks_api.executeTask('backup_local_db');
-        const backups_new = await utils.recFindByExt('appdata', 'bak');
-        const new_length = backups_new.length;
-        assert(original_length, new_length-1);
-    });
-
-    it('Check for missing files', async function() {
-        await db_api.removeAllRecords('files', {uid: 'test'});
-        const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
-        await db_api.insertRecordIntoTable('files', test_missing_file);
-        await tasks_api.executeTask('missing_files_check');
-        const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
-        assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
-    });
-
-    it('Check for duplicate files', async function() {
-        this.timeout(300000);
-        await db_api.removeAllRecords('files', {uid: 'test1'});
-        await db_api.removeAllRecords('files', {uid: 'test2'});
-        const test_duplicate_file1 = {uid: 'test1', path: 'test/missing_file.mp4'};
-        const test_duplicate_file2 = {uid: 'test2', path: 'test/missing_file.mp4'};
-        const test_duplicate_file3 = {uid: 'test3', path: 'test/missing_file.mp4'};
-        await db_api.insertRecordIntoTable('files', test_duplicate_file1);
-        await db_api.insertRecordIntoTable('files', test_duplicate_file2);
-        await db_api.insertRecordIntoTable('files', test_duplicate_file3);
-        await tasks_api.executeTask('duplicate_files_check');
-        const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
-        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
-        assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
-        assert(duplicated_record_count == 1, true);
-    });
-
-    it('Import unregistered files', async function() {
-        this.timeout(300000);
-
-        // pre-test cleanup
-        await db_api.removeAllRecords('files', {title: 'Sample File'});
-        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
-        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
-
-        // copies in files
-        fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
-        fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
-        await tasks_api.executeTask('missing_db_records');
-        const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
-        assert(!!imported_file, true);
-        
-        // post-test cleanup
-        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
-        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
-    });
-
-    it('Schedule and cancel task', async function() {
-        const today_4_hours = new Date();
-        today_4_hours.setHours(today_4_hours.getHours() + 4);
-        await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
-        await tasks_api.updateTaskSchedule('dummy_task', null);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], false);
-    });
-
-    it('Schedule and run task', async function() {
-        this.timeout(5000);
-        const today_1_second = new Date();
-        today_1_second.setSeconds(today_1_second.getSeconds() + 1);
-        await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
-        await utils.wait(2000);
-        const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
-        assert(dummy_task_obj['data'], true);
-    });
 });

backend/utils.js

@@ -1,13 +1,7 @@
-const fs = require('fs-extra');
-const path = require('path');
-const ffmpeg = require('fluent-ffmpeg');
-const archiver = require('archiver');
-const fetch = require('node-fetch');
-const ProgressBar = require('progress');
-
+const fs = require('fs-extra')
+const path = require('path')
 const config_api = require('./config');
-const logger = require('./logger');
-const CONSTS = require('./consts');
+const archiver = require('archiver');
 
 const is_windows = process.platform === 'win32';
 
@@ -48,7 +42,8 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
             files.push(jsonobj);
             continue;
         }
-        var upload_date = formatDateString(jsonobj.upload_date);
+        var upload_date = jsonobj.upload_date;
+        upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
 
         var isaudio = type === 'audio';
         var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
@@ -58,13 +53,13 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
     return files;
 }
 
-async function createContainerZipFile(file_name, container_file_objs) {
+async function createContainerZipFile(container_obj, container_file_objs) {
     const container_files_to_download = [];
     for (let i = 0; i < container_file_objs.length; i++) {
         const container_file_obj = container_file_objs[i];
         container_files_to_download.push(container_file_obj.path);
     }
-    return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
+    return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
 }
 
 async function createZipFile(zip_file_path, file_paths) {
@@ -146,7 +141,24 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
     return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(file_path) {
+function getDownloadedThumbnail(name, type, customPath = null) {
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
+
+    let jpgPath = path.join(customPath, name + '.jpg');
+    let webpPath = path.join(customPath, name + '.webp');
+    let pngPath = path.join(customPath, name + '.png');
+
+    if (fs.existsSync(jpgPath))
+        return jpgPath;
+    else if (fs.existsSync(webpPath))
+        return webpPath;
+    else if (fs.existsSync(pngPath))
+        return pngPath;
+    else
+        return null;
+}
+
+function getDownloadedThumbnail2(file_path, type) {
     const file_path_no_extension = removeFileExtension(file_path);
 
     let jpgPath = file_path_no_extension + '.jpg';
@@ -169,6 +181,10 @@ function getExpectedFileSize(input_info_jsons) {
 
     let expected_filesize = 0;
     info_jsons.forEach(info_json => {
+        if (info_json['filesize']) {
+            expected_filesize += info_json['filesize'];
+            return;
+        }
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
@@ -184,7 +200,29 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(file_path, type) {
+function fixVideoMetadataPerms(name, type, customPath = null) {
+    if (is_windows) return;
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
+                                                   : config_api.getConfigItem('ytdl_video_folder_path');
+
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+
+    const files_to_fix = [
+        // JSONs
+        path.join(customPath, name + '.info.json'),
+        path.join(customPath, name + ext + '.info.json'),
+        // Thumbnails
+        path.join(customPath, name + '.webp'),
+        path.join(customPath, name + '.jpg')
+    ];
+
+    for (const file of files_to_fix) {
+        if (!fs.existsSync(file)) continue;
+        fs.chmodSync(file, 0o644);
+    }
+}
+
+function fixVideoMetadataPerms2(file_path, type) {
     if (is_windows) return;
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
@@ -206,7 +244,19 @@ function fixVideoMetadataPerms(file_path, type) {
     }
 }
 
-function deleteJSONFile(file_path, type) {
+function deleteJSONFile(name, type, customPath = null) {
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
+                                                   : config_api.getConfigItem('ytdl_video_folder_path');
+
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+    let json_path = path.join(customPath, name + '.info.json');
+    let alternate_json_path = path.join(customPath, name + ext + '.info.json');
+
+    if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
+    if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
+}
+
+function deleteJSONFile2(file_path, type) {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
     const file_path_no_extension = removeFileExtension(file_path);
@@ -242,6 +292,7 @@ async function removeIDFromArchive(archive_path, id) {
     const updatedData = dataArray.join('\n');
     await fs.writeFile(archive_path, updatedData);
     if (line) return line;
+    if (err) throw err;
 }
 
 function durationStringToNumber(dur_str) {
@@ -264,12 +315,7 @@ function addUIDsToCategory(category, files) {
     return files_that_match;
 }
 
-function getCurrentDownloader() {
-    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
-    return details_json['downloader'];
-}
-
-async function recFindByExt(base, ext, files, result, recursive = true)
+async function recFindByExt(base,ext,files,result)
 {
     files = files || (await fs.readdir(base))
     result = result || []
@@ -278,7 +324,6 @@ async function recFindByExt(base, ext, files, result, recursive = true)
         var newbase = path.join(base,file)
         if ( (await fs.stat(newbase)).isDirectory() )
         {
-            if (!recursive) continue;
             result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
         }
         else
@@ -298,57 +343,6 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
-function formatDateString(date_string) {
-    return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
-}
-
-function createEdgeNGrams(str) {
-    if (str && str.length > 3) {
-        const minGram = 3
-        const maxGram = str.length
-        
-        return str.split(" ").reduce((ngrams, token) => {
-            if (token.length > minGram) {   
-                for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
-                    ngrams = [...ngrams, token.substr(0, i)]
-                }
-            } else {
-                ngrams = [...ngrams, token]
-            }
-            return ngrams
-        }, []).join(" ")
-    }
-    
-    return str
-}
-
-// ffmpeg helper functions
-
-async function cropFile(file_path, start, end, ext) {
-    return new Promise(resolve => {
-        const temp_file_path = `${file_path}.cropped${ext}`;
-        let base_ffmpeg_call = ffmpeg(file_path);
-        if (start) {
-            base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
-        }
-        if (end) {
-            base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
-        }
-        base_ffmpeg_call
-            .on('end', () => {
-                logger.verbose(`Cropping for '${file_path}' complete.`);
-                fs.unlinkSync(file_path);
-                fs.moveSync(temp_file_path, file_path);
-                resolve(true);
-            })
-            .on('error', (err) => {
-                logger.error(`Failed to crop ${file_path}.`);
-                logger.error(err);
-                resolve(false);
-            }).save(temp_file_path);
-    });
-}
-
 /**
  * setTimeout, but its a promise.
  * @param {number} ms
@@ -359,62 +353,6 @@ async function cropFile(file_path, start, end, ext) {
     });
 }
 
-async function checkExistsWithTimeout(filePath, timeout) {
-    return new Promise(function (resolve, reject) {
-
-        var timer = setTimeout(function () {
-            if (watcher) watcher.close();
-            reject(new Error('File did not exists and was not created during the timeout.'));
-        }, timeout);
-
-        fs.access(filePath, fs.constants.R_OK, function (err) {
-            if (!err) {
-                clearTimeout(timer);
-                if (watcher) watcher.close();
-                resolve();
-            }
-        });
-
-        var dir = path.dirname(filePath);
-        var basename = path.basename(filePath);
-        var watcher = fs.watch(dir, function (eventType, filename) {
-            if (eventType === 'rename' && filename === basename) {
-                clearTimeout(timer);
-                if (watcher) watcher.close();
-                resolve();
-            }
-        });
-    });
-}
-
-// helper function to download file using fetch
-async function fetchFile(url, path, file_label) {
-    var len = null;
-    const res = await fetch(url);
-
-    len = parseInt(res.headers.get("Content-Length"), 10);
-
-    var bar = new ProgressBar(`  Downloading ${file_label} [:bar] :percent :etas`, {
-        complete: '=',
-        incomplete: ' ',
-        width: 20,
-        total: len
-    });
-    const fileStream = fs.createWriteStream(path);
-    await new Promise((resolve, reject) => {
-        res.body.pipe(fileStream);
-        res.body.on("error", (err) => {
-          reject(err);
-        });
-        res.body.on('data', function (chunk) {
-            bar.tick(chunk.length);
-        });
-        fileStream.on("finish", function() {
-          resolve();
-        });
-    });
-}
-
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -440,23 +378,20 @@ module.exports = {
     getJSON: getJSON,
     getTrueFileName: getTrueFileName,
     getDownloadedThumbnail: getDownloadedThumbnail,
+    getDownloadedThumbnail2: getDownloadedThumbnail2,
     getExpectedFileSize: getExpectedFileSize,
     fixVideoMetadataPerms: fixVideoMetadataPerms,
+    fixVideoMetadataPerms2: fixVideoMetadataPerms2,
     deleteJSONFile: deleteJSONFile,
-    removeIDFromArchive: removeIDFromArchive,
+    deleteJSONFile2: deleteJSONFile2,
+    removeIDFromArchive, removeIDFromArchive,
     getDownloadedFilesByType: getDownloadedFilesByType,
     createContainerZipFile: createContainerZipFile,
     durationStringToNumber: durationStringToNumber,
     getMatchingCategoryFiles: getMatchingCategoryFiles,
     addUIDsToCategory: addUIDsToCategory,
-    getCurrentDownloader: getCurrentDownloader,
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
-    formatDateString: formatDateString,
-    cropFile: cropFile,
-    createEdgeNGrams: createEdgeNGrams,
     wait: wait,
-    checkExistsWithTimeout: checkExistsWithTimeout,
-    fetchFile: fetchFile,
     File: File
 }

backend/youtube-dl.js

@@ -1,127 +0,0 @@
-const fs = require('fs-extra');
-const fetch = require('node-fetch');
-
-const logger = require('./logger');
-const utils = require('./utils');
-const CONSTS = require('./consts');
-const config_api = require('./config.js');
-
-const is_windows = process.platform === 'win32';
-
-const download_sources = {
-    'youtube-dl': {
-        'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
-        'func': downloadLatestYoutubeDLBinary
-    },
-    'youtube-dlc': {
-        'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
-        'func': downloadLatestYoutubeDLCBinary
-    },
-    'yt-dlp': {
-        'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
-        'func': downloadLatestYoutubeDLPBinary
-    }
-}
-
-exports.checkForYoutubeDLUpdate = async () => {
-    return new Promise(async resolve => {
-        const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
-        const tags_url = download_sources[default_downloader]['tags_url'];
-        // get current version
-        let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
-        if (!current_app_details_exists) {
-            logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
-            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
-        }
-        let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
-        let current_version = current_app_details['version'];
-        let current_downloader = current_app_details['downloader'];
-        let stored_binary_path = current_app_details['path'];
-        if (!stored_binary_path || typeof stored_binary_path !== 'string') {
-            // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
-            const guessed_base_path = 'node_modules/youtube-dl/bin/';
-            const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
-            if (fs.existsSync(guessed_file_path)) {
-                stored_binary_path = guessed_file_path;
-                // logger.info('INFO: Guess successful! Update process continuing...')
-            } else {
-                logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
-                resolve(null);
-                return;
-            }
-        }
-
-        // got version, now let's check the latest version from the youtube-dl API
-
-
-        fetch(tags_url, {method: 'Get'})
-        .then(async res => res.json())
-        .then(async (json) => {
-            // check if the versions are different
-            if (!json || !json[0]) {
-                logger.error(`Failed to check ${default_downloader} version for an update.`)
-                resolve(null);
-                return;
-            }
-            const latest_update_version = json[0]['name'];
-            if (current_version !== latest_update_version || default_downloader !== current_downloader) {
-                // versions different or different downloader is being used, download new update
-                resolve(latest_update_version);
-            } else {
-                resolve(null);
-            }
-            return;
-        })
-        .catch(err => {
-            logger.error(`Failed to check ${default_downloader} version for an update.`)
-            logger.error(err);
-            resolve(null);
-            return;
-        });
-    });
-}
-
-exports.updateYoutubeDL = async (latest_update_version) => {
-    const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
-    await download_sources[default_downloader]['func'](latest_update_version);
-}
-
-async function downloadLatestYoutubeDLBinary(new_version) {
-    const file_ext = is_windows ? '.exe' : '';
-
-    const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
-    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
-
-    await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
-
-    updateDetailsJSON(new_version, 'youtube-dl');
-}
-
-async function downloadLatestYoutubeDLCBinary(new_version) {
-    const file_ext = is_windows ? '.exe' : '';
-
-    const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
-    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
-
-    await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
-
-    updateDetailsJSON(new_version, 'youtube-dlc');
-}
-
-async function downloadLatestYoutubeDLPBinary(new_version) {
-    const file_ext = is_windows ? '.exe' : '';
-
-    const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
-    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
-
-    await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
-
-    updateDetailsJSON(new_version, 'yt-dlp');
-}
-
-function updateDetailsJSON(new_version, downloader) {
-    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
-    if (new_version) details_json['version'] = new_version;
-    details_json['downloader'] = downloader;
-    fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
-}

docker-build.sh

@@ -1,43 +0,0 @@
-#!/bin/sh
-
-# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
-# and also optimizing some code with this commit.
-# xoxo :D
-
-case $(uname -m) in
-  x86_64)
-    ARCH=amd64;;
-  aarch64)
-    ARCH=arm64;;
-  armhf)
-    ARCH=armhf;;
-  armv7)
-    ARCH=armel;;
-  armv7l)
-    ARCH=armel;;
-  *)
-    echo "Unsupported architecture: $(uname -m)"
-    exit 1
-esac
-
-echo "(INFO) Architecture detected: $ARCH"
-echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
-apt-get update && apt-get -y install curl xz-utils
-echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
-curl -o ffmpeg.txz \
-    --connect-timeout 5 \
-    --max-time 10 \
-    --retry 5 \
-    --retry-delay 0 \
-    --retry-max-time 40 \
-    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
-mkdir /tmp/ffmpeg
-tar xf ffmpeg.txz -C /tmp/ffmpeg
-echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
-apt-get -y remove curl xz-utils
-apt-get -y autoremove
-echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
-cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
-cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
-echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
-rm -rf /tmp/ffmpeg ffmpeg.txz

docker-compose.yml

@@ -7,8 +7,6 @@ services:
             ytdl_use_local_db: 'false'
             write_ytdl_config: 'true'
         restart: always
-        depends_on:
-            - ytdl-mongo-db
         volumes:
             - ./appdata:/app/appdata
             - ./audio:/app/audio
@@ -17,7 +15,7 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:nightly
+        image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
         image: mongo
         ports:
@@ -25,6 +23,5 @@ services:
         logging:
             driver: "none"          
         container_name: mongo-db
-        restart: always
         volumes:
             - ./db/:/data/db

package.json

@@ -5,14 +5,12 @@
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
-    "build": "ng build --configuration production",
-    "prebuild": "node src/postbuild.mjs",
+    "build": "ng build",
     "heroku-postbuild": "npm install --prefix backend",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e",
-    "electron": "ng build --base-href ./ && electron .",
-    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
+    "electron": "ng build --base-href ./ && electron ."
   },
   "engines": {
     "node": "12.3.1",
@@ -20,62 +18,55 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
-    "@angular/animations": "^13.3.4",
-    "@angular/cdk": "^13.3.4",
-    "@angular/common": "^13.3.4",
-    "@angular/compiler": "^13.3.4",
-    "@angular/core": "^13.3.4",
-    "@angular/forms": "^13.3.4",
-    "@angular/localize": "^13.3.4",
-    "@angular/material": "^13.3.4",
-    "@angular/platform-browser": "^13.3.4",
-    "@angular/platform-browser-dynamic": "^13.3.4",
-    "@angular/router": "^13.3.4",
-    "@fontsource/material-icons": "^4.5.4",
+    "@angular-devkit/core": "^11.0.4",
+    "@angular/animations": "^11.0.4",
+    "@angular/cdk": "^11.0.2",
+    "@angular/common": "^11.0.4",
+    "@angular/compiler": "^11.0.4",
+    "@angular/core": "^11.0.4",
+    "@angular/forms": "^11.0.4",
+    "@angular/localize": "^11.0.4",
+    "@angular/material": "^11.0.2",
+    "@angular/platform-browser": "^11.0.4",
+    "@angular/platform-browser-dynamic": "^11.0.4",
+    "@angular/router": "^11.0.4",
     "@ngneat/content-loader": "^5.0.0",
-    "@videogular/ngx-videogular": "^5.0.1",
+    "@videogular/ngx-videogular": "^2.1.0",
     "core-js": "^2.4.1",
-    "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
     "filesize": "^6.1.0",
     "fingerprintjs2": "^2.1.0",
-    "fs-extra": "^10.0.0",
-    "material-icons": "^1.10.8",
+    "material-icons": "^0.5.4",
     "nan": "^2.14.1",
     "ng-lazyload-image": "^7.0.1",
-    "ngx-avatars": "^1.3.1",
-    "ngx-file-drop": "^13.0.0",
+    "ngx-avatar": "^4.0.0",
+    "ngx-file-drop": "^9.0.1",
     "rxjs": "^6.6.3",
     "rxjs-compat": "^6.0.0-rc.0",
     "tslib": "^2.0.0",
-    "typescript": "~4.6.3",
-    "xliff-to-json": "^1.0.4",
-    "zone.js": "~0.11.4"
+    "typescript": "~4.0.5",
+    "web-animations-js": "^2.3.2",
+    "zone.js": "~0.10.2"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
-    "@angular/cli": "^13.3.3",
-    "@angular/compiler-cli": "^13.3.4",
-    "@angular/language-service": "^13.3.4",
+    "@angular-devkit/build-angular": "^0.1100.4",
+    "@angular/cli": "^11.0.4",
+    "@angular/compiler-cli": "^11.0.4",
+    "@angular/language-service": "^11.0.4",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
     "@types/jasmine": "~3.6.0",
     "@types/node": "^12.11.1",
-    "@typescript-eslint/eslint-plugin": "^4.29.0",
-    "@typescript-eslint/parser": "^4.29.0",
     "codelyzer": "^6.0.0",
-    "electron": "^13.6.6",
-    "eslint": "^7.32.0",
+    "electron": "^8.0.1",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~6.3.16",
+    "karma": "~5.0.0",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
     "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
-    "openapi-typescript-codegen": "^0.21.0",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/_palette.scss

@@ -1,11 +1,11 @@
 /* Coolors Exported Palette - coolors.co/e8aeb7-b8e1ff-a9fff7-94fbab-82aba1 */
 
 /* HSL */
-$color1: hsla(351, 56%, 80%, 1);
-$softblue: hsla(205, 100%, 86%, 1);
-$color3: hsla(174, 100%, 83%, 1);
-$color4: hsla(133, 93%, 78%, 1);
-$color5: hsla(165, 20%, 59%, 1);
+$color1: hsla(351%, 56%, 80%, 1);
+$softblue: hsla(205%, 100%, 86%, 1);
+$color3: hsla(174%, 100%, 83%, 1);
+$color4: hsla(133%, 93%, 78%, 1);
+$color5: hsla(165%, 20%, 59%, 1);
 
 /* RGB */
 $color1: rgba(232, 174, 183, 1);

src/api-types/index.ts

@@ -1,111 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
-export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
-export type { body_19 } from './models/body_19';
-export type { body_20 } from './models/body_20';
-export type { Category } from './models/Category';
-export { CategoryRule } from './models/CategoryRule';
-export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermissionsRequest';
-export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
-export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
-export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
-export type { ConcurrentStream } from './models/ConcurrentStream';
-export type { Config } from './models/Config';
-export type { ConfigResponse } from './models/ConfigResponse';
-export type { CreateCategoryRequest } from './models/CreateCategoryRequest';
-export type { CreateCategoryResponse } from './models/CreateCategoryResponse';
-export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
-export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
-export type { CropFileSettings } from './models/CropFileSettings';
-export type { DatabaseFile } from './models/DatabaseFile';
-export { DBBackup } from './models/DBBackup';
-export type { DBInfoResponse } from './models/DBInfoResponse';
-export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
-export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
-export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
-export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
-export type { DeleteUserRequest } from './models/DeleteUserRequest';
-export type { Download } from './models/Download';
-export type { DownloadArchiveRequest } from './models/DownloadArchiveRequest';
-export type { DownloadFileRequest } from './models/DownloadFileRequest';
-export type { DownloadRequest } from './models/DownloadRequest';
-export type { DownloadResponse } from './models/DownloadResponse';
-export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
-export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
-export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
-export type { File } from './models/File';
-export { FileType } from './models/FileType';
-export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
-export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
-export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
-export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
-export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
-export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
-export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
-export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
-export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
-export type { GetDownloadRequest } from './models/GetDownloadRequest';
-export type { GetDownloadResponse } from './models/GetDownloadResponse';
-export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
-export type { GetFileFormatsResponse } from './models/GetFileFormatsResponse';
-export type { GetFileRequest } from './models/GetFileRequest';
-export type { GetFileResponse } from './models/GetFileResponse';
-export type { GetFullTwitchChatRequest } from './models/GetFullTwitchChatRequest';
-export type { GetFullTwitchChatResponse } from './models/GetFullTwitchChatResponse';
-export type { GetLogsRequest } from './models/GetLogsRequest';
-export type { GetLogsResponse } from './models/GetLogsResponse';
-export type { GetMp3sResponse } from './models/GetMp3sResponse';
-export type { GetMp4sResponse } from './models/GetMp4sResponse';
-export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
-export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
-export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
-export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
-export type { GetRolesResponse } from './models/GetRolesResponse';
-export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
-export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
-export type { GetTaskRequest } from './models/GetTaskRequest';
-export type { GetTaskResponse } from './models/GetTaskResponse';
-export type { GetUsersResponse } from './models/GetUsersResponse';
-export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
-export type { inline_response_200_15 } from './models/inline_response_200_15';
-export type { LoginRequest } from './models/LoginRequest';
-export type { LoginResponse } from './models/LoginResponse';
-export type { Playlist } from './models/Playlist';
-export type { RegisterRequest } from './models/RegisterRequest';
-export type { RegisterResponse } from './models/RegisterResponse';
-export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
-export { Schedule } from './models/Schedule';
-export type { SetConfigRequest } from './models/SetConfigRequest';
-export type { SharingToggle } from './models/SharingToggle';
-export type { SubscribeRequest } from './models/SubscribeRequest';
-export type { SubscribeResponse } from './models/SubscribeResponse';
-export type { Subscription } from './models/Subscription';
-export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
-export type { SuccessObject } from './models/SuccessObject';
-export type { TableInfo } from './models/TableInfo';
-export type { Task } from './models/Task';
-export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
-export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
-export type { TransferDBRequest } from './models/TransferDBRequest';
-export type { TransferDBResponse } from './models/TransferDBResponse';
-export type { TwitchChatMessage } from './models/TwitchChatMessage';
-export type { UnsubscribeRequest } from './models/UnsubscribeRequest';
-export type { UnsubscribeResponse } from './models/UnsubscribeResponse';
-export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
-export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
-export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
-export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
-export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
-export type { UpdaterStatus } from './models/UpdaterStatus';
-export type { UpdateServerRequest } from './models/UpdateServerRequest';
-export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
-export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
-export type { UpdateUserRequest } from './models/UpdateUserRequest';
-export type { User } from './models/User';
-export { UserPermission } from './models/UserPermission';
-export type { Version } from './models/Version';
-export type { VersionInfoResponse } from './models/VersionInfoResponse';
-export { YesNo } from './models/YesNo';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface AddFileToPlaylistRequest {
-    file_uid: string;
-    playlist_id: string;
-}

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -1,11 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { UserPermission } from './UserPermission';
-import { YesNo } from './YesNo';
-
-export interface BaseChangePermissionsRequest {
-    permission: UserPermission;
-    new_value: YesNo;
-}

src/api-types/models/Category.ts

@@ -1,15 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { CategoryRule } from './CategoryRule';
-
-export interface Category {
-    name?: string;
-    uid?: string;
-    rules?: Array<CategoryRule>;
-    /**
-     * Overrides file output for downloaded files in category
-     */
-    custom_output?: string;
-}

src/api-types/models/CategoryRule.ts

@@ -1,26 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface CategoryRule {
-    preceding_operator?: CategoryRule.preceding_operator;
-    comparator?: CategoryRule.comparator;
-}
-
-export namespace CategoryRule {
-
-    export enum preceding_operator {
-        OR = 'or',
-        AND = 'and',
-    }
-
-    export enum comparator {
-        INCLUDES = 'includes',
-        NOT_INCLUDES = 'not_includes',
-        EQUALS = 'equals',
-        NOT_EQUALS = 'not_equals',
-    }
-
-
-}

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
-
-export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
-    role: string;
-}

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
-
-export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
-    user_uid: string;
-}

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -1,11 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface CheckConcurrentStreamRequest {
-    /**
-     * UID of the concurrent stream
-     */
-    uid: string;
-}

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { ConcurrentStream } from './ConcurrentStream';
-
-export interface CheckConcurrentStreamResponse {
-    stream: ConcurrentStream;
-}

src/api-types/models/ConcurrentStream.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface ConcurrentStream {
-    playback_timestamp?: number;
-    unix_timestamp?: number;
-    playing?: boolean;
-}

src/api-types/models/Config.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface Config {
-    YoutubeDLMaterial: any;
-}

src/api-types/models/ConfigResponse.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Config } from './Config';
-
-export interface ConfigResponse {
-    config_file: Config;
-    success: boolean;
-}

src/api-types/models/CreateCategoryRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface CreateCategoryRequest {
-    name: string;
-}

src/api-types/models/CreateCategoryResponse.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Category } from './Category';
-
-export interface CreateCategoryResponse {
-    new_category?: Category;
-    success?: boolean;
-}

src/api-types/models/CreatePlaylistRequest.ts

@@ -1,12 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-
-export interface CreatePlaylistRequest {
-    playlistName: string;
-    uids: Array<string>;
-    type: FileType;
-    thumbnailURL: string;
-}

src/api-types/models/CreatePlaylistResponse.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Playlist } from './Playlist';
-
-export interface CreatePlaylistResponse {
-    new_playlist: Playlist;
-    success: boolean;
-}

src/api-types/models/CropFileSettings.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface CropFileSettings {
-    cropFileStart: number;
-    cropFileEnd: number;
-}

src/api-types/models/DBBackup.ts

@@ -1,21 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DBBackup {
-    name: string;
-    timestamp: number;
-    size: number;
-    source: DBBackup.source;
-}
-
-export namespace DBBackup {
-
-    export enum source {
-        LOCAL = 'local',
-        REMOTE = 'remote',
-    }
-
-
-}

src/api-types/models/DBInfoResponse.ts

@@ -1,18 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { TableInfo } from './TableInfo';
-
-export interface DBInfoResponse {
-    using_local_db?: boolean;
-    stats_by_table?: {
-files?: TableInfo,
-playlists?: TableInfo,
-categories?: TableInfo,
-subscriptions?: TableInfo,
-users?: TableInfo,
-roles?: TableInfo,
-download_queue?: TableInfo,
-};
-}

src/api-types/models/DatabaseFile.ts

@@ -1,22 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DatabaseFile {
-    id: string;
-    title: string;
-    thumbnailURL: string;
-    isAudio: boolean;
-    /**
-     * In seconds
-     */
-    duration: number;
-    url: string;
-    uploader: string;
-    size: number;
-    path: string;
-    upload_date: string;
-    uid: string;
-    sharingEnabled?: boolean;
-}

src/api-types/models/DeleteCategoryRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DeleteCategoryRequest {
-    category_uid: string;
-}

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DeleteMp3Mp4Request {
-    uid: string;
-    blacklistMode?: boolean;
-}

src/api-types/models/DeletePlaylistRequest.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-
-export interface DeletePlaylistRequest {
-    playlist_id: string;
-    type: FileType;
-}

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -1,15 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { SubscriptionRequestData } from './SubscriptionRequestData';
-
-export interface DeleteSubscriptionFileRequest {
-    file: string;
-    file_uid?: string;
-    sub: SubscriptionRequestData;
-    /**
-     * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
-     */
-    deleteForever?: boolean;
-}

src/api-types/models/DeleteUserRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DeleteUserRequest {
-    uid: string;
-}

src/api-types/models/Dictionary.ts

@@ -1,7 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-export type Dictionary<T> = {
-    [key: string]: T;
-}

src/api-types/models/Download.ts

@@ -1,26 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface Download {
-    uid: string;
-    ui_uid?: string;
-    running: boolean;
-    finished: boolean;
-    paused: boolean;
-    finished_step: boolean;
-    url: string;
-    type: string;
-    title: string;
-    step_index: number;
-    percent_complete: number;
-    timestamp_start: number;
-    /**
-     * Error text, set if download fails.
-     */
-    error?: string | null;
-    user_uid?: string;
-    sub_id?: string;
-    sub_name?: string;
-}

src/api-types/models/DownloadArchiveRequest.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DownloadArchiveRequest {
-    sub: {
-archive_dir: string,
-};
-}

src/api-types/models/DownloadFileRequest.ts

@@ -1,14 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-
-export interface DownloadFileRequest {
-    uid?: string;
-    uuid?: string;
-    sub_id?: string;
-    playlist_id?: string;
-    url?: string;
-    type?: FileType;
-}

src/api-types/models/DownloadRequest.ts

@@ -1,44 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { CropFileSettings } from './CropFileSettings';
-import { FileType } from './FileType';
-
-export interface DownloadRequest {
-    url: string;
-    /**
-     * Video format code. Overrides other quality options.
-     */
-    customQualityConfiguration?: string;
-    /**
-     * Custom command-line arguments for youtube-dl. Overrides all other options, except url.
-     */
-    customArgs?: string;
-    /**
-     * Additional command-line arguments for youtube-dl. Added to whatever args would normally be used.
-     */
-    additionalArgs?: string;
-    /**
-     * Custom output filename template.
-     */
-    customOutput?: string;
-    /**
-     * Login with this account ID
-     */
-    youtubeUsername?: string;
-    /**
-     * Account password
-     */
-    youtubePassword?: string;
-    /**
-     * Height of the video, if known
-     */
-    selectedHeight?: string;
-    /**
-     * Specify ffmpeg/avconv audio quality
-     */
-    maxBitrate?: string;
-    type?: FileType;
-    cropFileSettings?: CropFileSettings;
-}

src/api-types/models/DownloadResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Download } from './Download';
-
-export interface DownloadResponse {
-    download?: Download;
-}

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -1,23 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-import { Subscription } from './Subscription';
-
-export interface DownloadTwitchChatByVODIDRequest {
-    /**
-     * File ID
-     */
-    id: string;
-    /**
-     * ID of the VOD
-     */
-    vodId: string;
-    type: FileType;
-    /**
-     * User UID
-     */
-    uuid?: string;
-    sub?: Subscription;
-}

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { TwitchChatMessage } from './TwitchChatMessage';
-
-export interface DownloadTwitchChatByVODIDResponse {
-    chat: Array<TwitchChatMessage>;
-}

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface DownloadVideosForSubscriptionRequest {
-    subID: string;
-}

src/api-types/models/File.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface File {
-    id?: string;
-}

src/api-types/models/FileType.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export enum FileType {
-    AUDIO = 'audio',
-    VIDEO = 'video',
-}

src/api-types/models/GenerateArgsResponse.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GenerateArgsResponse {
-    args?: Array<string>;
-}

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GenerateNewApiKeyResponse {
-    new_api_key: string;
-}

src/api-types/models/GetAllCategoriesResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Category } from './Category';
-
-export interface GetAllCategoriesResponse {
-    categories: Array<Category>;
-}

src/api-types/models/GetAllDownloadsRequest.ts

@@ -1,11 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetAllDownloadsRequest {
-    /**
-     * Filters downloads with the array
-     */
-    uids?: Array<string> | null;
-}

src/api-types/models/GetAllDownloadsResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Download } from './Download';
-
-export interface GetAllDownloadsResponse {
-    downloads?: Array<Download>;
-}

src/api-types/models/GetAllFilesResponse.ts

@@ -1,14 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
-
-export interface GetAllFilesResponse {
-    files: Array<DatabaseFile>;
-    /**
-     * All video playlists
-     */
-    playlists: Array<Playlist>;
-}

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Subscription } from './Subscription';
-
-export interface GetAllSubscriptionsResponse {
-    subscriptions: Array<Subscription>;
-}

src/api-types/models/GetAllTasksResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Task } from './Task';
-
-export interface GetAllTasksResponse {
-    tasks?: Array<Task>;
-}

src/api-types/models/GetDBBackupsResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { DBBackup } from './DBBackup';
-
-export interface GetDBBackupsResponse {
-    tasks?: Array<DBBackup>;
-}

src/api-types/models/GetDownloadRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetDownloadRequest {
-    download_uid: string;
-}

src/api-types/models/GetDownloadResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { Download } from './Download';
-
-export interface GetDownloadResponse {
-    download?: Download;
-}

src/api-types/models/GetFileFormatsRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetFileFormatsRequest {
-    url?: string;
-}

src/api-types/models/GetFileFormatsResponse.ts

@@ -1,12 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { File } from './File';
-
-export interface GetFileFormatsResponse {
-    success: boolean;
-    result: {
-formats?: Array<any>,
-};
-}

src/api-types/models/GetFileRequest.ts

@@ -1,17 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-
-export interface GetFileRequest {
-    /**
-     * Video UID
-     */
-    uid: string;
-    type?: FileType;
-    /**
-     * User UID
-     */
-    uuid?: string;
-}

src/api-types/models/GetFileResponse.ts

@@ -1,10 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { DatabaseFile } from './DatabaseFile';
-
-export interface GetFileResponse {
-    success: boolean;
-    file?: DatabaseFile;
-}

src/api-types/models/GetFullTwitchChatRequest.ts

@@ -1,19 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-import { Subscription } from './Subscription';
-
-export interface GetFullTwitchChatRequest {
-    /**
-     * File ID
-     */
-    id: string;
-    type: FileType;
-    /**
-     * User UID
-     */
-    uuid?: string;
-    sub?: Subscription;
-}

src/api-types/models/GetFullTwitchChatResponse.ts

@@ -1,9 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetFullTwitchChatResponse {
-    success: boolean;
-    error?: string;
-}

src/api-types/models/GetLogsRequest.ts

@@ -1,8 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetLogsRequest {
-    lines?: number;
-}

src/api-types/models/GetLogsResponse.ts

@@ -1,12 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-
-export interface GetLogsResponse {
-    /**
-     * Number of lines to retrieve from the bottom
-     */
-    logs?: string;
-    success?: boolean;
-}

src/api-types/models/GetMp3sResponse.ts

@@ -1,14 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
-
-export interface GetMp3sResponse {
-    mp3s: Array<DatabaseFile>;
-    /**
-     * All audio playlists
-     */
-    playlists: Array<Playlist>;
-}

src/api-types/models/GetMp4sResponse.ts

@@ -1,14 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
-
-export interface GetMp4sResponse {
-    mp4s: Array<DatabaseFile>;
-    /**
-     * All video playlists
-     */
-    playlists: Array<Playlist>;
-}

src/api-types/models/GetPlaylistRequest.ts

@@ -1,12 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-
-export interface GetPlaylistRequest {
-    playlist_id: string;
-    type?: FileType;
-    uuid?: string;
-    include_file_metadata?: boolean;
-}

src/api-types/models/GetPlaylistResponse.ts

@@ -1,12 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-import { FileType } from './FileType';
-import { Playlist } from './Playlist';
-
-export interface GetPlaylistResponse {
-    playlist: Playlist;
-    type: FileType;
-    success: boolean;
-}