Merge branch 'master' into patch-1

.eslintrc.json

@@ -0,0 +1,20 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+    }
+}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment**
+- YoutubeDL-Material version
+- Docker tag: <tag> (optional)
+
+Ideally you'd copy the info as presented on the "About" dialogue
+in YoutubeDL-Material.
+(for that, click on the three dots on the top right and then
+check "installation details". On later versions of YoutubeDL-
+Material you will find pretty much all the crucial information
+here that we need in most cases!)
+
+**Additional context**
+Add any other context about the problem here. For example, a YouTube link.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[FEATURE]"
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/build.yml

@@ -15,15 +15,31 @@ jobs:
       - name: checkout code
         uses: actions/checkout@v2
       - name: setup node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          cache: 'npm'
       - name: install dependencies
         run: |
           npm install
           cd backend
           npm install
           sudo npm install -g @angular/cli
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
       - name: build
-        run: ng build --prod
+        run: npm run build
       - name: prepare artifact upload
         shell: pwsh
         run: |
@@ -35,7 +51,7 @@ jobs:
           Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
-          New-Item -Path ./build/youtubedl-material -Name users
+          New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
           Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
           Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
       - name: upload build artifact

.github/workflows/docker-release.yml

@@ -13,6 +13,19 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
       - name: setup multi-arch docker build

.github/workflows/docker.yml

@@ -10,6 +10,19 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
       - name: setup multi-arch docker build

.gitignore

@@ -65,4 +65,5 @@ backend/appdata/logs/error.log
 backend/appdata/users.json
 backend/users/*
 backend/appdata/cookies.txt
-backend/public
+backend/public
+src/assets/i18n/*.json

.vscode/tasks.json

@@ -0,0 +1,25 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "npm",
+			"script": "start",
+			"problemMatcher": [],
+			"label": "Dev: start frontend",
+			"detail": "ng serve"
+		},
+		{
+			"label": "Dev: start backend",
+			"type": "shell",
+			"command": "set YTDL_MODE=debug && node app.js",
+			"options": {
+				"cwd": "./backend"
+			},
+			"presentation": {
+				"reveal": "always",
+				"panel": "new"
+			},
+			"problemMatcher": []
+		}
+	]
+}

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12 as frontend
+FROM alpine:latest as frontend
 
 RUN apk add --no-cache \
   npm
@@ -11,28 +11,33 @@ RUN npm install
 
 COPY [ "angular.json", "tsconfig.json", "/build/" ]
 COPY [ "src/", "/build/src/" ]
-RUN ng build --prod
+RUN npm run build
 
 #--------------#
 
-FROM alpine:3.12
+FROM alpine:latest
 
 ENV UID=1000 \
   GID=1000 \
   USER=youtube
 
+ENV NO_UPDATE_NOTIFIER=true
+
 RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
 
 RUN apk add --no-cache \
   ffmpeg \
   npm \
   python2 \
+  python3 \
   su-exec \
   && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
     atomicparsley
 
 WORKDIR /app
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
+ENV PM2_HOME=/app/pm2
+RUN npm install pm2 -g
 RUN npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
@@ -40,4 +45,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "node", "app.js" ]
+CMD [ "pm2-runtime", "pm2.config.js" ]

README.md

@@ -29,7 +29,7 @@ NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker
 Debian/Ubuntu:
 
 ```bash
-sudo apt-get install nodejs youtube-dl ffmpeg
+sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
 ```
 
 CentOS 7:
@@ -67,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
 
 To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
 
-Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
+Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
 
 The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
 
@@ -77,6 +77,10 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ## Docker
 
+### Host-specific instructions
+
+If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+
 ### Setup
 
 If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
@@ -106,6 +110,12 @@ To get started, go to the settings menu and enable the public API from the *Extr
 
 Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above.
 
+## iOS Shortcut 
+
+If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?)
+
+You can download Shortcut [here.](https://routinehub.co/shortcut/10283/)
+
 ## Contributing
 
 If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away.
@@ -124,7 +134,7 @@ Official translators:
 * German - UnlimitedCookies
 * Chinese - TyRoyal
 
-See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
+See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
 
 ## License
 

backend/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "env": {
+        "node": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended"
+    ],
+    "parser": "esprima",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [],
+    "rules": {
+    },
+    "root": true
+}

backend/appdata/default.json

@@ -12,14 +12,16 @@
       "custom_args": "",
       "safe_download_override": false,
       "include_thumbnail": true,
-      "include_metadata": true
+      "include_metadata": true,
+      "max_concurrent_downloads": 5,
+      "download_rate_limit": ""
     },
     "Extra": {
       "title_top": "YoutubeDL-Material",
       "file_manager_enabled": true,
       "allow_quality_select": true,
       "download_only_mode": false,
-      "allow_multi_download_mode": true,
+      "allow_autoplay": true,
       "enable_downloads_manager": true,
       "allow_playlist_categorization": true
     },
@@ -30,7 +32,9 @@
       "youtube_API_key": "",
       "use_twitch_API": false,
       "twitch_API_key": "",
-      "twitch_auto_download_chat": false
+      "twitch_auto_download_chat": false,
+      "use_sponsorblock_API": false,
+      "generate_NFO_files": false
     },
     "Themes": {
       "default_theme": "default",
@@ -54,6 +58,10 @@
         "searchFilter": "(uid={{username}})"
       }
     },
+    "Database": {
+      "use_local_db": true,
+      "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
+    },
     "Advanced": {
       "default_downloader": "youtube-dl",
       "use_default_downloading_agent": true,
@@ -61,8 +69,8 @@
       "multi_user_mode": false,
       "allow_advanced_download": false,
       "use_cookies": false,
-      "jwt_expiration": 86400, 
+      "jwt_expiration": 86400,
       "logger_level": "info"
     }
   }
-}
+}

backend/authentication/auth.js

@@ -1,12 +1,10 @@
-const path = require('path');
 const config_api = require('../config');
 const consts = require('../consts');
-var subscriptions_api = require('../subscriptions')
+const logger = require('../logger');
-const fs = require('fs-extra');
-var jwt = require('jsonwebtoken');
-const { uuid } = require('uuidv4');
-var bcrypt = require('bcryptjs');
 
+const jwt = require('jsonwebtoken');
+const { uuid } = require('uuidv4');
+const bcrypt = require('bcryptjs');
 
 var LocalStrategy = require('passport-local').Strategy;
 var LdapStrategy = require('passport-ldapauth');
@@ -14,17 +12,14 @@ var JwtStrategy = require('passport-jwt').Strategy,
     ExtractJwt = require('passport-jwt').ExtractJwt;
 
 // other required vars
-let logger = null;
+let db_api = null;
-let db =  null;
-let users_db = null;
 let SERVER_SECRET = null;
 let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function(input_db, input_users_db, input_logger) {
+exports.initialize = function(db_api) {
-  setLogger(input_logger)
+  setDB(db_api);
-  setDB(input_db, input_users_db);
 
   /*************************
    * Authentication module
@@ -34,21 +29,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
 
   SERVER_SECRET = null;
-  if (users_db.get('jwt_secret').value()) {
+  if (db_api.users_db.get('jwt_secret').value()) {
-    SERVER_SECRET = users_db.get('jwt_secret').value();
+    SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
   } else {
     SERVER_SECRET = uuid();
-    users_db.set('jwt_secret', SERVER_SECRET).write();
+    db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
   }
 
   opts = {}
   opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
   opts.secretOrKey = SERVER_SECRET;
-  /*opts.issuer = 'example.com';
-  opts.audience = 'example.com';*/
 
-  exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) {
+  exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
-    const user = users_db.get('users').find({uid: jwt_payload.user}).value();
+    const user = await db_api.getRecord('users', {uid: jwt_payload.user});
     if (user) {
         return done(null, user);
     } else {
@@ -58,13 +51,8 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
   }));
 }
 
-function setLogger(input_logger) {
+function setDB(input_db_api) {
-  logger = input_logger;
+  db_api = input_db_api;
-}
-
-function setDB(input_db, input_users_db) {
-  db = input_db;
-  users_db = input_users_db;
 }
 
 exports.passport = require('passport');
@@ -80,7 +68,7 @@ exports.passport.deserializeUser(function(user, done) {
 /***************************************
  * Register user with hashed password
  **************************************/
-exports.registerUser = function(req, res) {
+exports.registerUser = async function(req, res) {
   var userid = req.body.userid;
   var username = req.body.username;
   var plaintextPassword = req.body.password;
@@ -98,20 +86,20 @@ exports.registerUser = function(req, res) {
   }
 
   bcrypt.hash(plaintextPassword, saltRounds)
-    .then(function(hash) {
+    .then(async function(hash) {
       let new_user = generateUserObject(userid, username, hash);
       // check if user exists
-      if (users_db.get('users').find({uid: userid}).value()) {
+      if (await db_api.getRecord('users', {uid: userid})) {
         // user id is taken!
         logger.error('Registration failed: UID is already taken!');
         res.status(409).send('UID is already taken!');
-      } else if (users_db.get('users').find({name: username}).value()) {
+      } else if (await db_api.getRecord('users', {name: username})) {
           // user name is taken!
           logger.error('Registration failed: User name is already taken!');
           res.status(409).send('User name is already taken!');
       } else {
         // add to db
-        users_db.get('users').push(new_user).write();
+        await db_api.insertRecordIntoTable('users', new_user);
         logger.verbose(`New user created: ${new_user.name}`);
         res.send({
           user: new_user
@@ -144,16 +132,18 @@ exports.registerUser = function(req, res) {
  ************************************************/
 
 
+exports.login = async (username, password) => {
+  const user = await db_api.getRecord('users', {name: username});
+  if (!user) { logger.error(`User ${username} not found`); return false }
+  if (user.auth_method && user.auth_method !== 'internal') { return false }
+  return await bcrypt.compare(password, user.passhash) ? user : false;
+}
+
 exports.passport.use(new LocalStrategy({
     usernameField: 'username',
     passwordField: 'password'},
     async function(username, password, done) {
-        const user = users_db.get('users').find({name: username}).value();
+      return done(null, await exports.login(username, password));
-        if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
-        if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
-        if (user) {
-            return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
-        }
     }
 ));
 
@@ -164,17 +154,17 @@ var getLDAPConfiguration = function(req, callback) {
 };
 
 exports.passport.use(new LdapStrategy(getLDAPConfiguration,
-  function(user, done) {
+  async function(user, done) {
     // check if ldap auth is enabled
     const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
     if (!ldap_enabled) return done(null, false);
 
     const user_uid = user.uid;
-    let db_user = users_db.get('users').find({uid: user_uid}).value();
+    let db_user = await db_api.getRecord('users', {uid: user_uid});
     if (!db_user) {
       // generate DB user
       let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
-      users_db.get('users').push(new_user).write();
+      await db_api.insertRecordIntoTable('users', new_user);
       db_user = new_user;
       logger.verbose(`Generated new user ${user_uid} using LDAP`);
     }
@@ -198,11 +188,11 @@ exports.generateJWT = function(req, res, next) {
   next();
 }
 
-exports.returnAuthResponse = function(req, res) {
+exports.returnAuthResponse = async function(req, res) {
   res.status(200).json({
     user: req.user,
     token: req.token,
-    permissions: exports.userPermissions(req.user.uid),
+    permissions: await exports.userPermissions(req.user.uid),
     available_permissions: consts['AVAILABLE_PERMISSIONS']
   });
 }
@@ -215,7 +205,7 @@ exports.returnAuthResponse = function(req, res) {
  * It also passes the user object to the next
  * middleware through res.locals
  **************************************/
-exports.ensureAuthenticatedElseError = function(req, res, next) {
+exports.ensureAuthenticatedElseError = (req, res, next) => {
   var token = getToken(req.query);
   if( token ) {
     try {
@@ -233,10 +223,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
 }
 
 // change password
-exports.changeUserPassword = async function(user_uid, new_pass) {
+exports.changeUserPassword = async (user_uid, new_pass) => {
   try {
     const hash = await bcrypt.hash(new_pass, saltRounds);
-    users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
+    await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
     return true;
   } catch (err) {
     return false;
@@ -244,16 +234,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
 }
 
 // change user permissions
-exports.changeUserPermissions = function(user_uid, permission, new_value) {
+exports.changeUserPermissions = async (user_uid, permission, new_value) => {
   try {
-    const user_db_obj = users_db.get('users').find({uid: user_uid});
+    await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
-    user_db_obj.get('permissions').pull(permission).write();
+    await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
-    user_db_obj.get('permission_overrides').pull(permission).write();
     if (new_value === 'yes') {
-      user_db_obj.get('permissions').push(permission).write();
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
-      user_db_obj.get('permission_overrides').push(permission).write();
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
     } else if (new_value === 'no') {
-      user_db_obj.get('permission_overrides').push(permission).write();
+      await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
     }
     return true;
   } catch (err) {
@@ -263,12 +252,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
 }
 
 // change role permissions
-exports.changeRolePermissions = function(role, permission, new_value) {
+exports.changeRolePermissions = async (role, permission, new_value) => {
   try {
-    const role_db_obj = users_db.get('roles').get(role);
+    await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
-    role_db_obj.get('permissions').pull(permission).write();
     if (new_value === 'yes') {
-      role_db_obj.get('permissions').push(permission).write();
+      await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
     }
     return true;
   } catch (err) {
@@ -277,19 +265,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
   }
 }
 
-exports.adminExists = function() {
+exports.adminExists = async function() {
-  return !!users_db.get('users').find({uid: 'admin'}).value();
+  return !!(await db_api.getRecord('users', {uid: 'admin'}));
 }
 
 // video stuff
 
-exports.getUserVideos = function(user_uid, type) {
+exports.getUserVideos = async function(user_uid, type) {
-    const user = users_db.get('users').find({uid: user_uid}).value();
+    const files = await db_api.getRecords('files', {user_uid: user_uid});
-    return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files'];
+    return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
 }
 
-exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
+exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
-  let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
+  let file = await db_api.getRecord('files', {file_uid: file_uid});
 
   // prevent unauthorized users from accessing the file info
   if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -297,58 +285,17 @@ exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
   return file;
 }
 
-exports.addPlaylist = function(user_uid, new_playlist) {
+exports.removePlaylist = async function(user_uid, playlistID) {
-  users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
+  await db_api.removeRecord('playlist', {playlistID: playlistID});
   return true;
 }
 
-exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
+exports.getUserPlaylists = async function(user_uid) {
-  users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
+  return await db_api.getRecords('playlists', {user_uid: user_uid});
-  return true;
 }
 
-exports.removePlaylist = function(user_uid, playlistID) {
+exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
-  users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write();
+  let playlist = await db_api.getRecord('playlists', {id: playlistID});
-  return true;
-}
-
-exports.getUserPlaylists = function(user_uid, user_files = null) {
-  const user = users_db.get('users').find({uid: user_uid}).value();
-  const playlists = JSON.parse(JSON.stringify(user['playlists']));
-  const categories = db.get('categories').value();
-  if (categories && user_files) {
-      categories.forEach(category => {
-          const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio);
-          const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio);
-          if (audio_files && audio_files.length > 0) {
-              playlists.push({
-                  name: category['name'],
-                  thumbnailURL: audio_files[0].thumbnailURL,
-                  thumbnailPath: audio_files[0].thumbnailPath,
-                  fileNames: audio_files.map(file => file.id),
-                  type: 'audio',
-                  uid: user_uid,
-                  auto: true
-              });
-          }
-          if (video_files && video_files.length > 0) {
-              playlists.push({
-                  name: category['name'],
-                  thumbnailURL: video_files[0].thumbnailURL,
-                  thumbnailPath: video_files[0].thumbnailPath,
-                  fileNames: video_files.map(file => file.id),
-                  type: 'video',
-                  uid: user_uid,
-                  auto: true
-              });
-          }
-      });
-  }
-  return playlists;
-}
-
-exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) {
-  let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value();
 
   // prevent unauthorized users from accessing the file info
   if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -356,109 +303,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false)
   return playlist;
 }
 
-exports.registerUserFile = function(user_uid, file_object) {
+exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
-  users_db.get('users').find({uid: user_uid}).get(`files`)
-    .remove({
-        path: file_object['path']
-    }).write();
-
-  users_db.get('users').find({uid: user_uid}).get(`files`)
-      .push(file_object)
-      .write();
-}
-
-exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) {
   let success = false;
-  const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
+  is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
-  if (file_obj) {
+  success = true;
-    const type = file_obj.isAudio ? 'audio' : 'video';
-    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-    const ext = type === 'audio' ? '.mp3' : '.mp4';
-
-    // close descriptors
-    if (config_api.descriptors[file_obj.id]) {
-      try {
-          for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) {
-            config_api.descriptors[file_obj.id][i].destroy();
-          }
-      } catch(e) {
-
-      }
-    }
-
-    const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext);
-    users_db.get('users').find({uid: user_uid}).get(`files`)
-      .remove({
-          uid: file_uid
-      }).write();
-    if (await fs.pathExists(full_path)) {
-      // remove json and file
-      const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
-      const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
-      let youtube_id = null;
-      if (await fs.pathExists(json_path)) {
-        youtube_id = await fs.readJSON(json_path).id;
-        await fs.unlink(json_path);
-      } else if (await fs.pathExists(alternate_json_path)) {
-        youtube_id = await fs.readJSON(alternate_json_path).id;
-        await fs.unlink(alternate_json_path);
-      }
-
-      await fs.unlink(full_path);
-
-      // do archive stuff
-
-      let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-      if (useYoutubeDLArchive) {
-          const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
-
-          // use subscriptions API to remove video from the archive file, and write it to the blacklist
-          if (await fs.pathExists(archive_path)) {
-              const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
-              if (blacklistMode && line) {
-                let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
-                // adds newline to the beginning of the line
-                line = '\n' + line;
-                await fs.appendFile(blacklistPath, line);
-              }
-          } else {
-              logger.info(`Could not find archive file for ${type} files. Creating...`);
-              await fs.ensureFile(archive_path);
-          }
-      }
-    }
-    success = true;
-  } else {
-    success = false;
-    logger.warn(`User file ${file_uid} does not exist!`);
-  }
-
   return success;
 }
 
-exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
+exports.userHasPermission = async function(user_uid, permission) {
-  let success = false;
-  const user_db_obj = users_db.get('users').find({uid: user_uid});
-  if (user_db_obj.value()) {
-    const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid});
-    if (file_db_obj.value()) {
-      success = true;
-      file_db_obj.assign({sharingEnabled: enabled}).write();
-    }
-  }
 
-  return success;
+  const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
-}
-
-exports.userHasPermission = function(user_uid, permission) {
-  const user_obj = users_db.get('users').find({uid: user_uid}).value();
   const role = user_obj['role'];
   if (!role) {
     // role doesn't exist
     logger.error('Invalid role ' + role);
     return false;
   }
-  const role_permissions = (users_db.get('roles').value())['permissions'];
+  const role_permissions = (await db_api.getRecords('roles'))['permissions'];
 
   const user_has_explicit_permission = user_obj['permissions'].includes(permission);
   const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -481,16 +342,17 @@ exports.userHasPermission = function(user_uid, permission) {
   }
 }
 
-exports.userPermissions = function(user_uid) {
+exports.userPermissions = async function(user_uid) {
   let user_permissions = [];
-  const user_obj = users_db.get('users').find({uid: user_uid}).value();
+  const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
   const role = user_obj['role'];
   if (!role) {
     // role doesn't exist
     logger.error('Invalid role ' + role);
     return null;
   }
-  const role_permissions = users_db.get('roles').get(role).get('permissions').value()
+  const role_obj = await db_api.getRecord('roles', {key: role});
+  const role_permissions = role_obj['permissions'];
 
   for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
     let permission = consts['AVAILABLE_PERMISSIONS'][i];

backend/categories.js

@@ -1,16 +1,12 @@
-const config_api = require('./config');
+const utils = require('./utils');
+const logger = require('./logger');
 
-var logger = null;
-var db = null;
-var users_db = null;
 var db_api = null;
 
-function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
+function setDB(input_db_api) { db_api = input_db_api }
-function setLogger(input_logger) { logger = input_logger; }
 
-function initialize(input_db, input_users_db, input_logger, input_db_api) {
+function initialize(input_db_api) {
-    setDB(input_db, input_users_db, input_db_api);
+    setDB(input_db_api);
-    setLogger(input_logger);
 }
 
 /*
@@ -38,10 +34,9 @@ async function categorize(file_jsons) {
     if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
 
     let selected_category = null;
-    const categories = getCategories();
+    const categories = await getCategories();
     if (!categories) {
-        logger.warn('Categories could not be found. Initializing categories...');
+        logger.warn('Categories could not be found.');
-        db.assign({categories: []}).write();
         return null;
     }
 
@@ -63,11 +58,29 @@ async function categorize(file_jsons) {
     return selected_category;
 }
 
-function getCategories() {
+async function getCategories() {
-    const categories = db.get('categories').value();
+    const categories = await db_api.getRecords('categories');
     return categories ? categories : null;
 }
 
+async function getCategoriesAsPlaylists(files = null) {
+    const categories_as_playlists = [];
+    const available_categories = await getCategories();
+    if (available_categories && files) {
+        for (let category of available_categories) {
+            const files_that_match = utils.addUIDsToCategory(category, files);
+            if (files_that_match && files_that_match.length > 0) {
+                category['thumbnailURL'] = files_that_match[0].thumbnailURL;
+                category['thumbnailPath'] = files_that_match[0].thumbnailPath;
+                category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
+                category['id'] = category['uid'];
+                categories_as_playlists.push(category);
+            }
+        }
+    }
+    return categories_as_playlists;
+}
+
 function applyCategoryRules(file_json, rules, category_name) {
     let rules_apply = false;
     for (let i = 0; i < rules.length; i++) {
@@ -78,10 +91,10 @@ function applyCategoryRules(file_json, rules, category_name) {
 
         switch (rule['comparator']) {
             case 'includes':
-                rule_applies = file_json[rule['property']].includes(rule['value']);
+                rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
                 break;
             case 'not_includes':
-                rule_applies = !(file_json[rule['property']].includes(rule['value']));
+                rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
                 break;
             case 'equals':
                 rule_applies = file_json[rule['property']] === rule['value'];
@@ -107,23 +120,25 @@ function applyCategoryRules(file_json, rules, category_name) {
     return rules_apply;
 }
 
-async function addTagToVideo(tag, video, user_uid) {
+// async function addTagToVideo(tag, video, user_uid) {
-    // TODO: Implement
+//     // TODO: Implement
-}
+// }
 
-async function removeTagFromVideo(tag, video, user_uid) {
+// async function removeTagFromVideo(tag, video, user_uid) {
-    // TODO: Implement
+//     // TODO: Implement
-}
+// }
 
-// adds tag to list of existing tags (used for tag suggestions)
+// // adds tag to list of existing tags (used for tag suggestions)
-async function addTagToExistingTags(tag) {
+// async function addTagToExistingTags(tag) {
-    const existing_tags = db.get('tags').value();
+//     const existing_tags = db.get('tags').value();
-    if (!existing_tags.includes(tag)) {
+//     if (!existing_tags.includes(tag)) {
-        db.get('tags').push(tag).write();
+//         db.get('tags').push(tag).write();
-    }
+//     }
-}
+// }
 
 module.exports = {
     initialize: initialize,
     categorize: categorize,
+    getCategories: getCategories,
+    getCategoriesAsPlaylists: getCategoriesAsPlaylists
 }

backend/config.js

@@ -1,3 +1,5 @@
+const logger = require('./logger');
+
 const fs = require('fs');
 
 let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
@@ -5,11 +7,7 @@ const debugMode = process.env.YTDL_MODE === 'debug';
 
 let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
 
-var logger = null;
+function initialize() {
-function setLogger(input_logger) { logger = input_logger; }
-
-function initialize(input_logger) {
-    setLogger(input_logger);
     ensureConfigFileExists();
     ensureConfigItemsExist();
 }
@@ -97,13 +95,13 @@ function getConfigItem(key) {
     }
     let path = CONFIG_ITEMS[key]['path'];
     const val = Object.byString(config_json, path);
-    if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
+    if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
         logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
         setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
         return Object.byString(DEFAULT_CONFIG, path);
     }
     return Object.byString(config_json, path);
-};
+}
 
 function setConfigItem(key, value) {
     let success = false;
@@ -175,7 +173,7 @@ module.exports = {
     globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
 }
 
-DEFAULT_CONFIG = {
+const DEFAULT_CONFIG = {
     "YoutubeDLMaterial": {
       "Host": {
         "url": "http://example.com",
@@ -189,14 +187,16 @@ DEFAULT_CONFIG = {
         "custom_args": "",
         "safe_download_override": false,
         "include_thumbnail": true,
-        "include_metadata": true
+        "include_metadata": true,
+        "max_concurrent_downloads": 5,
+        "download_rate_limit": ""
       },
       "Extra": {
         "title_top": "YoutubeDL-Material",
         "file_manager_enabled": true,
         "allow_quality_select": true,
         "download_only_mode": false,
-        "allow_multi_download_mode": true,
+        "allow_autoplay": true,
         "enable_downloads_manager": true,
         "allow_playlist_categorization": true
       },
@@ -207,7 +207,9 @@ DEFAULT_CONFIG = {
         "youtube_API_key": "",
         "use_twitch_API": false,
         "twitch_API_key": "",
-        "twitch_auto_download_chat": false
+        "twitch_auto_download_chat": false,
+        "use_sponsorblock_API": false,
+        "generate_NFO_files": false
       },
       "Themes": {
         "default_theme": "default",
@@ -216,7 +218,7 @@ DEFAULT_CONFIG = {
       "Subscriptions": {
         "allow_subscriptions": true,
         "subscriptions_base_path": "subscriptions/",
-        "subscriptions_check_interval": "300",
+        "subscriptions_check_interval": "86400",
         "redownload_fresh_uploads": false
       },
       "Users": {
@@ -231,6 +233,10 @@ DEFAULT_CONFIG = {
             "searchFilter": "(uid={{username}})"
         }
       },
+      "Database": {
+        "use_local_db": true,
+        "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
+      },
       "Advanced": {
         "default_downloader": "youtube-dl",
         "use_default_downloading_agent": true,

backend/consts.js

@@ -1,4 +1,4 @@
-let CONFIG_ITEMS = {
+exports.CONFIG_ITEMS = {
     // Host
     'ytdl_url': {
         'key': 'ytdl_url',
@@ -42,6 +42,14 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_include_metadata',
         'path': 'YoutubeDLMaterial.Downloader.include_metadata'
     },
+    'ytdl_max_concurrent_downloads': {
+        'key': 'ytdl_max_concurrent_downloads',
+        'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
+    },
+    'ytdl_download_rate_limit': {
+        'key': 'ytdl_download_rate_limit',
+        'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
+    },
 
     // Extra
     'ytdl_title_top': {
@@ -60,9 +68,9 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_download_only_mode',
         'path': 'YoutubeDLMaterial.Extra.download_only_mode'
     },
-    'ytdl_allow_multi_download_mode': {
+    'ytdl_allow_autoplay': {
-        'key': 'ytdl_allow_multi_download_mode',
+        'key': 'ytdl_allow_autoplay',
-        'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
+        'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
     },
     'ytdl_enable_downloads_manager': {
         'key': 'ytdl_enable_downloads_manager',
@@ -102,6 +110,15 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
     },
+    'ytdl_use_sponsorblock_api': {
+        'key': 'ytdl_use_sponsorblock_api',
+        'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
+    },
+    'ytdl_generate_nfo_files': {
+        'key': 'ytdl_generate_nfo_files',
+        'path': 'YoutubeDLMaterial.API.generate_NFO_files'
+    },
+
 
     // Themes
     'ytdl_default_theme': {
@@ -126,10 +143,6 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_subscriptions_check_interval',
         'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
     },
-    'ytdl_subscriptions_check_interval': {
-        'key': 'ytdl_subscriptions_check_interval',
-        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
-    },
     'ytdl_subscriptions_redownload_fresh_uploads': {
         'key': 'ytdl_subscriptions_redownload_fresh_uploads',
         'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
@@ -153,6 +166,16 @@ let CONFIG_ITEMS = {
         'path': 'YoutubeDLMaterial.Users.ldap_config'
     },
 
+    // Database
+    'ytdl_use_local_db': {
+        'key': 'ytdl_use_local_db',
+        'path': 'YoutubeDLMaterial.Database.use_local_db'
+    },
+    'ytdl_mongodb_connection_string': {
+        'key': 'ytdl_mongodb_connection_string',
+        'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
+    },
+
     // Advanced
     'ytdl_default_downloader': {
         'key': 'ytdl_default_downloader',
@@ -188,7 +211,7 @@ let CONFIG_ITEMS = {
     }
 };
 
-AVAILABLE_PERMISSIONS = [
+exports.AVAILABLE_PERMISSIONS = [
     'filemanager',
     'settings',
     'subscriptions',
@@ -197,8 +220,6 @@ AVAILABLE_PERMISSIONS = [
     'downloads_manager'
 ];
 
-module.exports = {
+exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
-    CONFIG_ITEMS: CONFIG_ITEMS,
+
-    AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS,
+exports.CURRENT_VERSION = 'v4.2';
-    CURRENT_VERSION: 'v4.2'
-}

backend/downloader.js

@@ -0,0 +1,632 @@
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+const path = require('path');
+const mergeFiles = require('merge-files');
+const NodeID3 = require('node-id3')
+const glob = require('glob')
+const Mutex = require('async-mutex').Mutex;
+
+const youtubedl = require('youtube-dl');
+
+const logger = require('./logger');
+const config_api = require('./config');
+const twitch_api = require('./twitch');
+const { create } = require('xmlbuilder2');
+const categories_api = require('./categories');
+const utils = require('./utils');
+
+let db_api = null;
+
+const mutex = new Mutex();
+let should_check_downloads = true;
+
+const archivePath = path.join(__dirname, 'appdata', 'archives');
+
+function setDB(input_db_api) { db_api = input_db_api }
+
+exports.initialize = (input_db_api) => {
+    setDB(input_db_api);
+    categories_api.initialize(db_api);
+    if (db_api.database_initialized) {
+        setupDownloads();
+    } else {
+        db_api.database_initialized_bs.subscribe(init => {
+            if (init) setupDownloads();
+        });
+    }
+}
+
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+    return await mutex.runExclusive(async () => {
+        const download = {
+            url: url,
+            type: type,
+            title: '',
+            user_uid: user_uid,
+            sub_id: sub_id,
+            sub_name: sub_name,
+            options: options,
+            uid: uuid(),
+            step_index: 0,
+            paused: false,
+            running: false,
+            finished_step: true,
+            error: null,
+            percent_complete: null,
+            finished: false,
+            timestamp_start: Date.now()
+        };
+        await db_api.insertRecordIntoTable('download_queue', download);
+    
+        should_check_downloads = true;
+        return download;
+    });
+}
+
+exports.pauseDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        logger.warn(`Download ${download_uid} is already paused!`);
+        return false;
+    } else if (download['finished']) {
+        logger.info(`Download ${download_uid} could not be paused before completing.`);
+        return false;
+    }
+
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
+}
+
+exports.resumeDownload = async (download_uid) => {
+    return await mutex.runExclusive(async () => {
+        const download = await db_api.getRecord('download_queue', {uid: download_uid});
+        if (!download['paused']) {
+            logger.warn(`Download ${download_uid} is not paused!`);
+            return false;
+        }
+
+        const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
+        should_check_downloads = true;
+        return success;
+    })
+}
+
+exports.restartDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    await exports.clearDownload(download_uid);
+    const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
+    
+    should_check_downloads = true;
+    return success;
+}
+
+exports.cancelDownload = async (download_uid) => {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['cancelled']) {
+        logger.warn(`Download ${download_uid} is already cancelled!`);
+        return false;
+    } else if (download['finished']) {
+        logger.info(`Download ${download_uid} could not be cancelled before completing.`);
+        return false;
+    }
+    return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
+}
+
+exports.clearDownload = async (download_uid) => {
+    return await db_api.removeRecord('download_queue', {uid: download_uid});
+}
+
+async function handleDownloadError(download_uid, error_message) {
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
+}
+
+async function setupDownloads() {
+    await fixDownloadState();
+    setInterval(checkDownloads, 1000);
+}
+
+async function fixDownloadState() {
+    const downloads = await db_api.getRecords('download_queue');
+    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
+    const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
+    for (let i = 0; i < running_downloads.length; i++) {
+        const running_download = running_downloads[i];
+        const update_obj = {finished_step: true, paused: true, running: false};
+        if (running_download['step_index'] > 0) {
+            update_obj['step_index'] = running_download['step_index'] - 1;
+        }
+        await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
+    }
+}
+
+async function checkDownloads() {
+    if (!should_check_downloads) return;
+
+    const downloads = await db_api.getRecords('download_queue');
+    downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
+
+    await mutex.runExclusive(async () => {
+        // avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
+        const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
+        if (running_downloads.length === 0) {
+            should_check_downloads = false;
+            logger.verbose('Disabling checking downloads as none are available.');
+        }
+        return;
+    });
+
+    let running_downloads_count = downloads.filter(download => download['running']).length;
+    const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
+    for (let i = 0; i < waiting_downloads.length; i++) {
+        const waiting_download = waiting_downloads[i];
+        const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
+        if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
+
+        if (waiting_download['finished_step'] && !waiting_download['finished']) {
+            // move to next step
+            running_downloads_count++;
+            if (waiting_download['step_index'] === 0) {
+                collectInfo(waiting_download['uid']);
+            } else if (waiting_download['step_index'] === 1) {
+                downloadQueuedFile(waiting_download['uid']);
+            }
+        }
+    }
+}
+
+async function collectInfo(download_uid) {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        return;
+    }
+    logger.verbose(`Collecting info for download ${download_uid}`);
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
+
+    const url = download['url'];
+    const type = download['type'];
+    const options = download['options'];
+
+    if (download['user_uid'] && !options.customFileFolderPath) {
+        let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        const user_path = path.join(usersFileFolder, download['user_uid'], type);
+        options.customFileFolderPath = user_path + path.sep;
+    }
+
+    let args = await exports.generateArgs(url, type, options, download['user_uid']);
+
+    // get video info prior to download
+    let info = await getVideoInfoByURL(url, args, download_uid);
+
+    if (!info) {
+        // info failed, error presumably already recorded
+        return;
+    }
+
+    let category = null;
+
+    // check if it fits into a category. If so, then get info again using new args
+    if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
+
+    // set custom output if the category has one and re-retrieve info so the download manager has the right file name
+    if (category && category['custom_output']) {
+        options.customOutput = category['custom_output'];
+        options.noRelativePath = true;
+        args = await exports.generateArgs(url, type, options, download['user_uid']);
+        info = await getVideoInfoByURL(url, args, download_uid);
+    }
+
+    // setup info required to calculate download progress
+
+    const expected_file_size = utils.getExpectedFileSize(info);
+
+    const files_to_check_for_progress = [];
+
+    // store info in download for future use
+    if (Array.isArray(info)) {
+        for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
+    } else {
+        files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
+    }
+
+    const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
+    await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
+                                                                    finished_step: true,
+                                                                    running: false,
+                                                                    options: options,
+                                                                    files_to_check_for_progress: files_to_check_for_progress,
+                                                                    expected_file_size: expected_file_size,
+                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                });
+}
+
+async function downloadQueuedFile(download_uid) {
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    if (download['paused']) {
+        return;
+    }
+    logger.verbose(`Downloading ${download_uid}`);
+    return new Promise(async resolve => {
+        const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+        const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
+
+        const url = download['url'];
+        const type = download['type'];
+        const options = download['options'];
+        const args = download['args'];
+        const category = download['category'];
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        if (options.customFileFolderPath) {
+            fileFolderPath = options.customFileFolderPath;
+        }
+        fs.ensureDirSync(fileFolderPath);
+
+        const start_time = Date.now();
+
+        const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
+
+        // download file
+        youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
+            const file_objs = [];
+            let end_time = Date.now();
+            let difference = (end_time - start_time)/1000;
+            logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
+            clearInterval(download_checker);
+            if (err) {
+                logger.error(err.stderr);
+                await handleDownloadError(download_uid, err.stderr);
+                resolve(false);
+                return;
+            } else if (output) {
+                if (output.length === 0 || output[0].length === 0) {
+                    // ERROR!
+                    const error_message = `No output received for video download, check if it exists in your archive.`;
+                    await handleDownloadError(download_uid, error_message);
+                    logger.warn(error_message);
+                    resolve(false);
+                    return;
+                }
+
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+
+                    if (!output_json) {
+                        continue;
+                    }
+
+                    // get filepath with no extension
+                    const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
+
+                    const ext = type === 'audio' ? '.mp3' : '.mp4';
+                    var full_file_path = filepath_no_extension + ext;
+                    var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
+
+                    if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
+                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                            let vodId = url.split('twitch.tv/videos/')[1];
+                            vodId = vodId.split('?')[0];
+                            twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
+                    }
+
+                    // renames file if necessary due to bug
+                    if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
+                        try {
+                            fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
+                            logger.info('Renamed ' + file_name + '.webm to ' + file_name);
+                        } catch(e) {
+                            logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
+                        }
+                    }
+
+                    if (type === 'audio') {
+                        let tags = {
+                            title: output_json['title'],
+                            artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
+                        }
+                        let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
+                        if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
+                    }
+
+                    if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
+                        exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
+                    }
+
+                    if (options.cropFileSettings) {
+                        await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
+                    }
+
+                    // registers file in DB
+                    const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
+
+                    file_objs.push(file_obj);
+                }
+
+                if (options.merged_string !== null && options.merged_string !== undefined) {
+                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
+                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
+                    const diff = current_merged_archive.replace(options.merged_string, '');
+                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+                    fs.appendFileSync(archive_path, diff);
+                }
+
+                let container = null;
+
+                if (file_objs.length > 1) {
+                    // create playlist
+                    const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                } else if (file_objs.length === 1) {
+                    container = file_objs[0];
+                } else {
+                    const error_message = 'Downloaded file failed to result in metadata object.';
+                    logger.error(error_message);
+                    await handleDownloadError(download_uid, error_message);
+                }
+
+                const file_uids = file_objs.map(file_obj => file_obj.uid);
+                await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
+                resolve();
+            }
+        });
+    });
+}
+
+// helper functions
+
+exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+    const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+
+    const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+    const globalArgs = config_api.getConfigItem('ytdl_custom_args');
+    const useCookies = config_api.getConfigItem('ytdl_use_cookies');
+    const is_audio = type === 'audio';
+
+    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+
+    if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
+
+    const customArgs = options.customArgs;
+    let customOutput = options.customOutput;
+    const customQualityConfiguration = options.customQualityConfiguration;
+
+    // video-specific args
+    const selectedHeight = options.selectedHeight;
+
+    // audio-specific args
+    const maxBitrate = options.maxBitrate;
+
+    const youtubeUsername = options.youtubeUsername;
+    const youtubePassword = options.youtubePassword;
+
+    let downloadConfig = null;
+    let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
+    const is_youtube = url.includes('youtu');
+    if (!is_audio && !is_youtube) {
+        // tiktok videos fail when using the default format
+        qualityPath = null;
+    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
+        qualityPath = ['-f', 'bestvideo+bestaudio']
+    }
+
+    if (customArgs) {
+        downloadConfig = customArgs.split(',,');
+    } else {
+        if (customQualityConfiguration) {
+            qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
+        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
+            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+        } else if (is_audio) {
+            qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
+        }
+
+        if (customOutput) {
+            customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
+            downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
+        } else {
+            downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
+        }
+
+        if (qualityPath) downloadConfig.push(...qualityPath);
+
+        if (is_audio && !options.skip_audio_args) {
+            downloadConfig.push('-x');
+            downloadConfig.push('--audio-format', 'mp3');
+        }
+
+        if (youtubeUsername && youtubePassword) {
+            downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
+        }
+
+        if (useCookies) {
+            if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
+                downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
+            } else {
+                logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
+            }
+        }
+
+        const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
+        const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
+        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
+            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
+        }
+
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
+            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
+
+            await fs.ensureDir(archive_folder);
+            await fs.ensureFile(archive_path);
+
+            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
+            await fs.ensureFile(blacklist_path);
+
+            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
+            await fs.ensureFile(merged_path);
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            await mergeFiles(inputPathList, merged_path);
+
+            options.merged_string = await fs.readFile(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
+        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+            downloadConfig.push('--write-thumbnail');
+        }
+
+        if (globalArgs && globalArgs !== '') {
+            // adds global args
+            if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
+                // if global args has an output, replce the original output with that of global args
+                const original_output_index = downloadConfig.indexOf('-o');
+                downloadConfig.splice(original_output_index, 2);
+            }
+            downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
+        }
+
+        if (options.additionalArgs && options.additionalArgs !== '') {
+            downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
+        }
+
+        const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
+        if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
+            downloadConfig.push('-r', rate_limit);
+        }
+        
+        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+        if (default_downloader === 'yt-dlp') {
+            downloadConfig.push('--no-clean-infojson');
+        }
+
+    }
+
+    // filter out incompatible args
+    downloadConfig = filterArgs(downloadConfig, is_audio);
+
+    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    return downloadConfig;
+}
+
+async function getVideoInfoByURL(url, args = [], download_uid = null) {
+    return new Promise(resolve => {
+        // remove bad args
+        const new_args = [...args];
+
+        const archiveArgIndex = new_args.indexOf('--download-archive');
+        if (archiveArgIndex !== -1) {
+            new_args.splice(archiveArgIndex, 2);
+        }
+
+        new_args.push('--dump-json');
+
+        youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
+            if (output) {
+                let outputs = [];
+                try {
+                    for (let i = 0; i < output.length; i++) {
+                        let output_json = null;
+                        try {
+                            output_json = JSON.parse(output[i]);
+                        } catch(e) {
+                            output_json = null;
+                        }
+    
+                        if (!output_json) {
+                            continue;
+                        }
+
+                        outputs.push(output_json);
+                    }
+                    resolve(outputs.length === 1 ? outputs[0] : outputs);
+                } catch(e) {
+                    const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
+                    logger.error(error);
+                    if (download_uid) {
+                        await handleDownloadError(download_uid, error);
+                    }
+                    resolve(null);
+                }
+            } else {
+                let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
+                if (err.stderr) error_message += `\n\n${err.stderr}`;
+                logger.error(error_message);
+                if (download_uid) {
+                    await handleDownloadError(download_uid, error_message);
+                }
+                resolve(null);
+            }
+        });
+    });
+}
+
+function filterArgs(args, isAudio) {
+    const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
+    const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
+    const args_to_remove = isAudio ? video_only_args : audio_only_args;
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+async function checkDownloadPercent(download_uid) {
+    /*
+    This is more of an art than a science, we're just selecting files that start with the file name,
+    thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
+
+    Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
+    be divided by the "total expected bytes."
+    */
+
+    const download = await db_api.getRecord('download_queue', {uid: download_uid});
+    const files_to_check_for_progress = download['files_to_check_for_progress'];
+    const resulting_file_size = download['expected_file_size'];
+
+    if (!resulting_file_size) return;
+
+    let sum_size = 0;
+    glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
+        files.forEach(async file => {
+            try {
+                const file_stats = fs.statSync(file);
+                if (file_stats && file_stats.size) {
+                    sum_size += file_stats.size;
+                }
+            } catch (e) {
+
+            }
+        });
+        const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+        await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
+    });
+}
+
+exports.generateNFOFile = (info, output_path) => {
+    const nfo_obj = {
+        episodedetails: {
+            title: info['fulltitle'],
+            episode: info['playlist_index'] ? info['playlist_index'] : undefined,
+            premiered: utils.formatDateString(info['upload_date']),
+            plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
+            director: info['artist'] ? info['artist'] : info['uploader']
+        }
+    };
+    const doc = create(nfo_obj);
+    const xml = doc.end({ prettyPrint: true });
+    fs.writeFileSync(output_path, xml);
+}
+
+function getArchiveFolder(fileFolderPath, options, user_uid) {
+    if (options.customArchivePath) {
+        return path.join(options.customArchivePath);
+    } else if (user_uid) {
+        return path.join(fileFolderPath, 'archives');
+    } else {
+        return path.join(archivePath);
+    }
+}

backend/ecosystem.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  apps : [{
+    name   : "YoutubeDL-Material",
+    script : "./app.js",
+    watch  : "placeholder",
+    watch_delay: 5000
+  }]
+}

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="node app.js"
+CMD="pm2-runtime pm2.config.js"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then

backend/logger.js

@@ -0,0 +1,23 @@
+const winston = require('winston');
+
+let debugMode = process.env.YTDL_MODE === 'debug';
+
+const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
+    return `${timestamp} ${level.toUpperCase()}: ${message}`;
+});
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(winston.format.timestamp(), defaultFormat),
+    defaultMeta: {},
+    transports: [
+      //
+      // - Write to all logs with level `info` and below to `combined.log`
+      // - Write all logs error (and below) to `error.log`.
+      //
+      new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
+      new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
+      new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
+    ]
+});
+
+module.exports = logger;

backend/package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon -q app.js"
+    "start": "nodemon app.js",
+    "debug": "set YTDL_MODE=debug && node app.js"
   },
   "nodemonConfig": {
     "ignore": [
@@ -14,7 +15,8 @@
       "public/*"
     ],
     "watch": [
-      "restart.json"
+      "restart_update.json",
+      "restart_general.json"
     ]
   },
   "repository": {
@@ -30,7 +32,8 @@
   "dependencies": {
     "archiver": "^3.1.1",
     "async": "^3.1.0",
-    "axios": "^0.21.1",
+    "async-mutex": "^0.3.1",
+    "axios": "^0.21.2",
     "bcryptjs": "^2.4.0",
     "compression": "^1.7.4",
     "config": "^3.2.3",
@@ -43,11 +46,13 @@
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
     "merge-files": "^0.1.2",
+    "mocha": "^8.4.0",
     "moment": "^2.29.1",
+    "mongodb": "^3.6.9",
     "multer": "^1.4.2",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^2.6.7",
     "node-id3": "^0.1.14",
-    "nodemon": "^2.0.2",
+    "nodemon": "^2.0.7",
     "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.0",
@@ -56,10 +61,12 @@
     "progress": "^2.0.3",
     "ps-node": "^0.1.6",
     "read-last-lines": "^1.7.2",
+    "rxjs": "^7.3.0",
     "shortid": "^2.2.15",
     "unzipper": "^0.10.10",
     "uuidv4": "^6.0.6",
-    "winston": "^3.2.1",
+    "winston": "^3.3.3",
+    "xmlbuilder2": "^3.0.2",
     "youtube-dl": "^3.0.2"
   }
 }

backend/pm2.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+    apps : [{
+      name   : "YoutubeDL-Material",
+      script : "./app.js",
+      watch  : "placeholder",
+      out_file: "/dev/null",
+      error_file: "/dev/null"
+    }]
+}

backend/subscriptions.js

@@ -1,27 +1,21 @@
-const FileSync = require('lowdb/adapters/FileSync')
+const fs = require('fs-extra');
+const path = require('path');
+const youtubedl = require('youtube-dl');
 
-var fs = require('fs-extra');
-const { uuid } = require('uuidv4');
-var path = require('path');
-
-var youtubedl = require('youtube-dl');
 const config_api = require('./config');
-const twitch_api = require('./twitch');
+const utils = require('./utils');
-var utils = require('./utils');
+const logger = require('./logger');
 
 const debugMode = process.env.YTDL_MODE === 'debug';
 
-var logger = null;
+let db_api = null;
-var db = null;
+let downloader_api = null;
-var users_db = null;
-var db_api = null;
 
-function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
+function setDB(input_db_api) { db_api = input_db_api }
-function setLogger(input_logger) { logger = input_logger; }
 
-function initialize(input_db, input_users_db, input_logger, input_db_api) {
+function initialize(input_db_api, input_downloader_api) {
-    setDB(input_db, input_users_db, input_db_api);
+    setDB(input_db_api);
-    setLogger(input_logger);
+    downloader_api = input_downloader_api;
 }
 
 async function subscribe(sub, user_uid = null) {
@@ -34,12 +28,7 @@ async function subscribe(sub, user_uid = null) {
         sub.isPlaylist = sub.url.includes('playlist');
         sub.videos = [];
 
-        let url_exists = false;
+        let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
-
-        if (user_uid)
-            url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
-        else
-            url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
 
         if (!sub.name && url_exists) {
             logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -48,23 +37,16 @@ async function subscribe(sub, user_uid = null) {
             return;
         }
 
-        // add sub to db
+        sub['user_uid'] = user_uid ? user_uid : undefined;
-        let sub_db = null;
+        await db_api.insertRecordIntoTable('subscriptions', sub);
-        if (user_uid) {
+
-            users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
+        let success = await getSubscriptionInfo(sub);
-            sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
-        } else {
-            db.get('subscriptions').push(sub).write();
-            sub_db = db.get('subscriptions').find({id: sub.id});
-        }
-        let success = await getSubscriptionInfo(sub, user_uid);
 
         if (success) {
-            sub = sub_db.value();
             getVideosForSub(sub, user_uid);
         } else {
             logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
-        };
+        }
 
         result_obj.success = success;
         result_obj.sub = sub;
@@ -73,13 +55,7 @@ async function subscribe(sub, user_uid = null) {
 
 }
 
-async function getSubscriptionInfo(sub, user_uid = null) {
+async function getSubscriptionInfo(sub) {
-    let basePath = null;
-    if (user_uid)
-        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
-    else
-        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
-
     // get videos
     let downloadConfig = ['--dump-json', '--playlist-end', '1'];
     let useCookies = config_api.getConfigItem('ytdl_use_cookies');
@@ -91,8 +67,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
         }
     }
 
-    return new Promise(resolve => {
+    return new Promise(async resolve => {
-        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, function(err, output) {
+        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
             if (debugMode) {
                 logger.info('Subscribe: got info for subscription ' + sub.id);
             }
@@ -122,31 +98,10 @@ async function getSubscriptionInfo(sub, user_uid = null) {
                         }
                         // if it's now valid, update
                         if (sub.name) {
-                            if (user_uid)
+                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
-                                users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
-                            else
-                                db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
                         }
                     }
 
-                    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-                    if (useArchive && !sub.archive) {
-                        // must create the archive
-                        const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
-                        const archive_path = path.join(archive_dir, 'archive.txt');
-
-                        // creates archive directory and text file if it doesn't exist
-                        fs.ensureDirSync(archive_dir);
-                        fs.ensureFileSync(archive_path);
-
-                        // updates subscription
-                        sub.archive = archive_dir;
-                        if (user_uid)
-                            users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
-                        else
-                            db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
-                    }
-
                     // TODO: get even more info
 
                     resolve(true);
@@ -163,13 +118,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
-    let result_obj = { success: false, error: '' };
 
     let id = sub.id;
-    if (user_uid)
+
-        users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
+    const sub_files = await db_api.getRecords('files', {sub_id: id});
-    else
+    for (let i = 0; i < sub_files.length; i++) {
-        db.get('subscriptions').remove({id: id}).write();
+        const sub_file = sub_files[i];
+        if (config_api.descriptors[sub_file['uid']]) {
+            try {
+                for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
+                    config_api.descriptors[sub_file['uid']][i].destroy();
+                }
+            } catch(e) {
+                continue;
+            }
+        }
+    }
+
+    await db_api.removeRecord('subscriptions', {id: id});
+    await db_api.removeAllRecords('files', {sub_id: id});
 
     // failed subs have no name, on unsubscribe they shouldn't error
     if (!sub.name) {
@@ -191,20 +158,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
 }
 
 async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
+    // TODO: combine this with deletefile
     let basePath = null;
-    let sub_db = null;
+    basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
-    if (user_uid) {
+                        : config_api.getConfigItem('ytdl_subscriptions_base_path');
-        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
-        sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
-    } else {
-        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
-        sub_db = db.get('subscriptions').find({id: sub.id});
-    }
     const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
-    sub_db.get('videos').remove({uid: file_uid}).write();
+
+    await db_api.removeRecord('files', {uid: file_uid});
 
     let filePath = appendedBasePath;
     const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
@@ -243,7 +206,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
                 const archive_path = path.join(sub.archive, 'archive.txt')
                 // if archive exists, remove line with video ID
                 if (await fs.pathExists(archive_path)) {
-                    await removeIDFromArchive(archive_path, retrievedID);
+                    utils.removeIDFromArchive(archive_path, retrievedID);
                 }
             }
             return true;
@@ -255,14 +218,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
 }
 
 async function getVideosForSub(sub, user_uid = null) {
-    // get sub_db
+    const latest_sub_obj = await getSubscription(sub.id);
-    let sub_db = null;
-    if (user_uid)
-        sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
-    else
-        sub_db = db.get('subscriptions').find({id: sub.id});
-
-    const latest_sub_obj = sub_db.value();
     if (!latest_sub_obj || latest_sub_obj['downloading']) {
         return false;
     }
@@ -277,42 +233,39 @@ async function getVideosForSub(sub, user_uid = null) {
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
     let appendedBasePath = getAppendedBasePath(sub, basePath);
-
+    fs.ensureDirSync(appendedBasePath);
-    let multiUserMode = null;
-    if (user_uid) {
-        multiUserMode = {
-            user: user_uid,
-            file_path: appendedBasePath
-        }
-    }
 
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose('Subscription: getting videos for subscription ' + sub.name);
+    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
 
-    return new Promise(resolve => {
+    return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
+            // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
                 if (err.stderr.includes('This video is unavailable')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        // TODO: reimplement
-                        for (let i = 0; i < outputs.length; i++) {
+
-                            const output = JSON.parse(outputs[i]);
+                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
-                            handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
+                        // for (let i = 0; i < outputs.length; i++) {
-                            if (err.stderr.includes(output['id']) && archive_path) {
+                        //     const output = JSON.parse(outputs[i]);
-                                // we found a video that errored! add it to the archive to prevent future errors
+                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                                if (sub.archive) {
+                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                                    archive_dir = sub.archive;
+                        //         // we found a video that errored! add it to the archive to prevent future errors
-                                    archive_path = path.join(archive_dir, 'archive.txt')
+                        //         if (sub.archive) {
-                                    fs.appendFileSync(archive_path, output['id']);
+                        //             archive_dir = sub.archive;
-                                }
+                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                            }
+                        //             fs.appendFileSync(archive_path, output['id']);
-                        }
+                        //         }
+                        //     }
+                        // }
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
@@ -325,21 +278,30 @@ async function getVideosForSub(sub, user_uid = null) {
                     resolve(true);
                     return;
                 }
+                const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
                         output_json = JSON.parse(output[i]);
+                        output_jsons.push(output_json);
                     } catch(e) {
                         output_json = null;
                     }
                     if (!output_json) {
                         continue;
                     }
-
-                    const reset_videos = i === 0;
-                    handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
                 }
 
+                const files_to_download = await getFilesToDownload(sub, output_jsons);
+                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+                for (let j = 0; j < files_to_download.length; j++) {
+                    const file_to_download = files_to_download[j];
+                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
+                }
+
+                resolve(files_to_download);
+
                 if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
                     await setFreshUploads(sub, user_uid);
                     checkVideosForFreshUploads(sub, user_uid);
@@ -354,6 +316,26 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+function generateOptionsForSubscriptionDownload(sub, user_uid) {
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+
+    const base_download_options = {
+        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        customFileFolderPath: getAppendedBasePath(sub, basePath),
+        customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
+        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        additionalArgs: sub.custom_args
+    }
+
+    return base_download_options;
+}
+
 async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
     // get basePath
     let basePath = null;
@@ -366,14 +348,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
-    let fullOutput = `'${appendedBasePath}/%(title)s.%(ext)s'`;
+    const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+
+    let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
     if (desired_path) {
-        fullOutput = `'${desired_path}.%(ext)s'`;
+        fullOutput = `"${desired_path}.%(ext)s"`;
     } else if (sub.custom_output) {
-        fullOutput = `'${appendedBasePath}/${sub.custom_output}.%(ext)s'`;
+        fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
     }
 
-    let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
+    let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
 
     let qualityPath = null;
     if (sub.type && sub.type === 'audio') {
@@ -388,7 +372,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     downloadConfig.push(...qualityPath)
 
     if (sub.custom_args) {
-        customArgsArray = sub.custom_args.split(',,');
+        const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
             // if custom args has a custom quality, replce the original quality with that of custom args
             const original_output_index = downloadConfig.indexOf('-f');
@@ -430,109 +414,72 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push('--write-thumbnail');
     }
 
+    const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
+    if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
+        downloadConfig.push('-r', rate_limit);
+    }
+
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+    if (default_downloader === 'yt-dlp') {
+        downloadConfig.push('--no-clean-infojson');
+    }
+
     return downloadConfig;
 }
 
-function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
+async function getFilesToDownload(sub, output_jsons) {
-    if (sub.streamingOnly) {
+    const files_to_download = [];
-        if (reset_videos) {
+    for (let i = 0; i < output_jsons.length; i++) {
-            sub_db.assign({videos: []}).write();
+        const output_json = output_jsons[i];
-        }
+        const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
-
+        if (file_missing) {
-        // remove unnecessary info
+            const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
-        output_json.formats = null;
+            if (file_with_path_exists) {
-
+                // or maybe just overwrite???
-        // add to db
+                logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
-        sub_db.get('videos').push(output_json).write();
+            }
-    } else {
+            files_to_download.push(output_json);
-        path_object = path.parse(output_json['_filename']);
-        const path_string = path.format(path_object);
-
-        if (sub_db.get('videos').find({path: path_string}).value()) {
-            // file already exists in DB, return early to avoid reseting the download date
-            return;
-        }
-
-        db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
-        const url = output_json['webpage_url'];
-        if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-            && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
-                const file_name = path.basename(output_json['_filename']);
-                const id = file_name.substring(0, file_name.length-4);
-                let vodId = url.split('twitch.tv/videos/')[1];
-                vodId = vodId.split('?')[0];
-                twitch_api.downloadTwitchChatByVODID(vodId, id, sub.type, multiUserMode.user, sub);
         }
     }
+    return files_to_download;
 }
 
-function getSubscriptions(user_uid = null) {
+
-    if (user_uid)
+async function getSubscriptions(user_uid = null) {
-        return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
+    return await db_api.getRecords('subscriptions', {user_uid: user_uid});
-    else
-        return db.get('subscriptions').value();
 }
 
-function getAllSubscriptions() {
+async function getAllSubscriptions() {
-    let subscriptions = null;
+    const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
-    if (multiUserMode) {
+    return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
-        subscriptions = [];
-        let users = users_db.get('users').value();
-        for (let i = 0; i < users.length; i++) {
-            if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']);
-        }
-    } else {
-        subscriptions = getSubscriptions();
-    }
-    return subscriptions;
 }
 
-function getSubscription(subID, user_uid = null) {
+async function getSubscription(subID) {
-    if (user_uid)
+    return await db_api.getRecord('subscriptions', {id: subID});
-        return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
-    else
-        return db.get('subscriptions').find({id: subID}).value();
 }
 
-function getSubscriptionByName(subName, user_uid = null) {
+async function getSubscriptionByName(subName, user_uid = null) {
-    if (user_uid)
+    return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
-        return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value();
-    else
-        return db.get('subscriptions').find({name: subName}).value();
 }
 
-function updateSubscription(sub, user_uid = null) {
+async function updateSubscription(sub) {
-    if (user_uid) {
+    await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
-        users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
-    } else {
-        db.get('subscriptions').find({id: sub.id}).assign(sub).write();
-    }
     return true;
 }
 
-function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
+async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
-    subs.forEach(sub => {
+    subs.forEach(async sub => {
-        updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
+        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
     });
 }
 
-function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
+async function updateSubscriptionProperty(sub, assignment_obj) {
-    if (user_uid) {
+    // TODO: combine with updateSubscription
-        users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
+    await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
-    } else {
-        db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
-    }
     return true;
 }
 
-function subExists(subID, user_uid = null) {
-    if (user_uid)
-        return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
-    else
-        return !!db.get('subscriptions').find({id: subID}).value();
-}
-
 async function setFreshUploads(sub, user_uid) {
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
     sub.videos.forEach(async video => {
@@ -548,7 +495,7 @@ async function checkVideosForFreshUploads(sub, user_uid) {
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
     sub.videos.forEach(async video => {
         if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
-            checkVideoIfBetterExists(video, sub, user_uid)
+            await checkVideoIfBetterExists(video, sub, user_uid)
         }
     });
 }
@@ -558,7 +505,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
     logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
     // simulate a download to verify that a better version exists
-    youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
+    youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
         if (err) {
             // video is not available anymore for whatever reason
         } else if (output) {
@@ -582,37 +529,9 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
 // helper functions
 
 function getAppendedBasePath(sub, base_path) {
-
     return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
 }
 
-async function removeIDFromArchive(archive_path, id) {
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
-    if (!data) {
-        logger.error('Archive could not be found.');
-        return;
-    }
-
-    let dataArray = data.split('\n'); // convert file data in an array
-    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-    let lastIndex = -1; // let say, we have not found the keyword
-
-    for (let index=0; index<dataArray.length; index++) {
-        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-            lastIndex = index; // found a line includes a id keyword
-            break;
-        }
-    }
-
-    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-
-    // UPDATE FILE WITH NEW DATA
-    const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
-    if (line) return line;
-    if (err) throw err;
-}
-
 module.exports = {
     getSubscription        : getSubscription,
     getSubscriptionByName  : getSubscriptionByName,
@@ -623,8 +542,7 @@ module.exports = {
     unsubscribe            : unsubscribe,
     deleteSubscriptionFile : deleteSubscriptionFile,
     getVideosForSub        : getVideosForSub,
-    removeIDFromArchive    : removeIDFromArchive,
-    setLogger              : setLogger,
     initialize             : initialize,
-    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple
+    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
+    generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
 }

backend/test/tests.js

@@ -0,0 +1,352 @@
+var assert = require('assert');
+const low = require('lowdb')
+var winston = require('winston');
+
+process.chdir('./backend')
+
+const FileSync = require('lowdb/adapters/FileSync');
+
+const adapter = new FileSync('./appdata/db.json');
+const db = low(adapter)
+
+const users_adapter = new FileSync('./appdata/users.json');
+const users_db = low(users_adapter);
+
+const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
+    return `${timestamp} ${level.toUpperCase()}: ${message}`;
+});
+
+let debugMode = process.env.YTDL_MODE === 'debug';
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(winston.format.timestamp(), defaultFormat),
+    defaultMeta: {},
+    transports: [
+      //
+      // - Write to all logs with level `info` and below to `combined.log`
+      // - Write all logs error (and below) to `error.log`.
+      //
+      new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
+      new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
+      new winston.transports.Console({level: 'debug', name: 'console'})
+    ]
+});
+
+var auth_api = require('../authentication/auth');
+var db_api = require('../db');
+const utils = require('../utils');
+const subscriptions_api = require('../subscriptions');
+const fs = require('fs-extra');
+const { uuid } = require('uuidv4');
+
+db_api.initialize(db, users_db);
+
+
+describe('Database', async function() {
+    describe('Import', async function() {
+        it('Migrate', async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords();
+            const success = await db_api.importJSONToDB(db.value(), users_db.value());
+            assert(success);
+        });
+
+        it('Transfer to remote', async function() {
+            await db_api.removeAllRecords('test');
+            await db_api.insertRecordIntoTable('test', {test: 'test'});
+
+            await db_api.transferDB(true);
+            const success = await db_api.getRecord('test', {test: 'test'});
+            assert(success);
+        });
+
+        it('Transfer to local', async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords('test');
+            await db_api.insertRecordIntoTable('test', {test: 'test'});
+
+            await db_api.transferDB(false);
+            const success = await db_api.getRecord('test', {test: 'test'});
+            assert(success);
+        });
+    });
+
+    describe('Export', function() {
+
+    });
+
+
+    describe('Basic functions', async function() {
+        beforeEach(async function() {
+            await db_api.connectToDB();
+            await db_api.removeAllRecords('test');
+        });
+        it('Add and read record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
+            const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
+            assert(added_record['test_add'] === 'test');
+            await db_api.removeRecord('test', {test_add: 'test'});
+        });
+
+        it('Update record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_update: 'test'});
+            await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
+            const updated_record = await db_api.getRecord('test', {test_update: 'test'});
+            assert(updated_record['added_field']);
+            await db_api.removeRecord('test', {test_update: 'test'});
+        });
+
+        it('Remove record', async function() {
+            await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
+            const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
+            assert(delete_succeeded);
+            const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
+            assert(!deleted_record);
+        });
+
+        it('Push to record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
+            await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 1);
+        });
+
+        it('Pull from record array', async function() {
+            await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
+            await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
+            const record = await db_api.getRecord('test', {test: 'test'});
+            assert(record);
+            assert(record['test_array'].length === 0);
+        });
+
+        it('Bulk add', async function() {
+            const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
+            const test_records = [];
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                test_records.push({
+                    uid: uuid()
+                });
+            }
+            const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+
+            const received_records = await db_api.getRecords('test');
+            assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
+        });
+
+        it('Bulk update', async function() {
+            // bulk add records
+            const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
+            const test_records = [];
+            const update_obj = {};
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                const test_uid =  uuid();
+                test_records.push({
+                    uid: test_uid
+                });
+                update_obj[test_uid] = {added_field: true};
+            }
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            assert(success);
+
+            // makes sure they are added
+            const received_records = await db_api.getRecords('test');
+            assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
+
+            success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
+            assert(success);
+
+            const received_updated_records = await db_api.getRecords('test');
+            for (let i = 0; i < received_updated_records.length; i++) {
+                success &= received_updated_records[i]['added_field'];
+            }
+            assert(success);
+        });
+
+        it('Stats', async function() {
+            const stats = await db_api.getDBStats();
+            assert(stats);
+        });
+
+        it('Query speed', async function() {
+            this.timeout(120000); 
+            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+            const test_records = [];
+            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                const uid = uuid();
+                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J  - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon:  http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+            }
+            const insert_start = Date.now();
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            const insert_end = Date.now();
+
+            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+            const query_start = Date.now();
+            const random_record = await db_api.getRecord('test', {uid: random_uid});
+            const query_end = Date.now();
+
+            console.log(random_record)
+
+            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+            success = !!random_record;
+
+            assert(success);
+        });
+    });
+});
+
+describe('Multi User', async function() {
+    let user = null;
+    const user_to_test = 'admin';
+    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
+    const playlist_to_test = 'ysabVZz4x';
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        auth_api.initialize(db_api, logger);
+        subscriptions_api.initialize(db_api, logger);
+        user = await auth_api.login('admin', 'pass');
+    });
+    describe('Authentication', function() {
+        it('login', async function() {
+            assert(user);
+        });
+    });
+    describe('Video player - normal', function() {
+        const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
+        it('Get video', async function() {
+            const video_obj = db_api.getVideo(video_to_test, 'admin');
+            assert(video_obj);
+        });
+
+        it('Video access - disallowed', async function() {
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            assert(!video_obj);
+        });
+
+        it('Video access - allowed', async function() {
+            await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
+            const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+            assert(video_obj);
+        });
+    });
+    describe('Zip generators', function() {
+        it('Playlist zip generator', async function() {
+            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
+            assert(playlist);
+            const playlist_files_to_download = [];
+            for (let i = 0; i < playlist['uids'].length; i++) {
+                const uid = playlist['uids'][i];
+                const playlist_file = await db_api.getVideo(uid, user_to_test);
+                playlist_files_to_download.push(playlist_file);
+            }
+            const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+
+        it('Subscription zip generator', async function() {
+            const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
+            const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
+            assert(sub);
+            const sub_files_to_download = [];
+            for (let i = 0; i < sub_videos.length; i++) {
+                const sub_file = sub_videos[i];
+                sub_files_to_download.push(sub_file);
+            }
+            const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+    });
+    // describe('Video player - subscription', function() {
+    //     const sub_to_test = '';
+    //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
+    //     it('Get video', async function() {
+    //         const video_obj = db_api.getVideo(video_to_test, 'admin', );
+    //         assert(video_obj);
+    //     });
+
+    //     it('Video access - disallowed', async function() {
+    //         await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test);
+    //         const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+    //         assert(!video_obj);
+    //     });
+
+    //     it('Video access - allowed', async function() {
+    //         await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test);
+    //         const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
+    //         assert(video_obj);
+    //     });
+    // });
+
+});
+    
+describe('Downloader', function() {
+    const downloader_api = require('../downloader');
+    downloader_api.initialize(db_api);
+    const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
+    const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
+    const options = {
+        ui_uid: uuid(),
+        user: 'admin'
+    }
+
+    beforeEach(async function() {
+        await db_api.connectToDB();
+        await db_api.removeAllRecords('download_queue');
+    });
+
+    it('Get file info', async function() {
+
+    });
+
+    it('Download file', async function() {
+        this.timeout(300000); 
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        console.log(returned_download);
+        await utils.wait(20000);
+
+    });
+
+    it('Queue file', async function() {
+        this.timeout(300000); 
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        console.log(returned_download);
+        await utils.wait(20000);
+    });
+
+    it('Pause file', async function() {
+
+    });
+
+    it('Generate args', async function() {
+        const args = await downloader_api.generateArgs(url, 'video', options);
+        console.log(args);
+    });
+
+    it('Generate args - subscription', async function() {
+        subscriptions_api.initialize(db_api, logger);
+        const sub = await subscriptions_api.getSubscription(sub_id);
+        const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
+        const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        console.log(args);
+    });
+
+    it('Generate kodi NFO file', async function() {
+        const nfo_file_path = './test/sample.nfo';
+        if (fs.existsSync(nfo_file_path)) {
+            fs.unlinkSync(nfo_file_path);
+        }
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        downloader_api.generateNFOFile(sample_json, nfo_file_path);
+        assert(fs.existsSync(nfo_file_path), true);
+    });
+});

backend/utils.js

@@ -1,6 +1,10 @@
-var fs = require('fs-extra')
+const fs = require('fs-extra')
-var path = require('path')
+const path = require('path')
+const ffmpeg = require('fluent-ffmpeg');
 const config_api = require('./config');
+const logger = require('./logger');
+const CONSTS = require('./consts')
+const archiver = require('archiver');
 
 const is_windows = process.platform === 'win32';
 
@@ -41,8 +45,7 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
             files.push(jsonobj);
             continue;
         }
-        var upload_date = jsonobj.upload_date;
+        var upload_date = formatDateString(jsonobj.upload_date);
-        upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
 
         var isaudio = type === 'audio';
         var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
@@ -52,6 +55,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
     return files;
 }
 
+async function createContainerZipFile(container_obj, container_file_objs) {
+    const container_files_to_download = [];
+    for (let i = 0; i < container_file_objs.length; i++) {
+        const container_file_obj = container_file_objs[i];
+        container_files_to_download.push(container_file_obj.path);
+    }
+    return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
+}
+
+async function createZipFile(zip_file_path, file_paths) {
+    let output = fs.createWriteStream(zip_file_path);
+
+    var archive = archiver('zip', {
+        gzip: true,
+        zlib: { level: 9 } // Sets the compression level.
+    });
+
+    archive.on('error', function(err) {
+        logger.error(err);
+        throw err;
+    });
+
+    // pipe archive data to the output file
+    archive.pipe(output);
+
+    for (let file_path of file_paths) {
+        const file_name = path.parse(file_path).base;
+        archive.file(file_path, {name: file_name})
+    }
+
+    await archive.finalize();
+
+    // wait a tiny bit for the zip to reload in fs
+    await wait(100);
+    return zip_file_path;
+}
+
 function getJSONMp4(name, customPath, openReadPerms = false) {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -84,16 +124,31 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
     return obj;
 }
 
+function getJSON(file_path, type) {
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+    let obj = null;
+    var jsonPath = removeFileExtension(file_path) + '.info.json';
+    var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
+    if (fs.existsSync(jsonPath))
+    {
+        obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+    } else if (fs.existsSync(alternateJsonPath)) {
+        obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
+    }
+    else obj = 0;
+    return obj;
+}
+
 function getJSONByType(type, name, customPath, openReadPerms = false) {
     return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
 }
 
-function getDownloadedThumbnail(name, type, customPath = null) {
+function getDownloadedThumbnail(file_path) {
-    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
+    const file_path_no_extension = removeFileExtension(file_path);
 
-    let jpgPath = path.join(customPath, name + '.jpg');
+    let jpgPath = file_path_no_extension + '.jpg';
-    let webpPath = path.join(customPath, name + '.webp');
+    let webpPath = file_path_no_extension + '.webp';
-    let pngPath = path.join(customPath, name + '.png');
+    let pngPath = file_path_no_extension + '.png';
 
     if (fs.existsSync(jpgPath))
         return jpgPath;
@@ -111,10 +166,6 @@ function getExpectedFileSize(input_info_jsons) {
 
     let expected_filesize = 0;
     info_jsons.forEach(info_json => {
-        if (info_json['filesize']) {
-            expected_filesize += info_json['filesize'];
-            return;
-        }
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
@@ -130,20 +181,20 @@ function getExpectedFileSize(input_info_jsons) {
     return expected_filesize;
 }
 
-function fixVideoMetadataPerms(name, type, customPath = null) {
+function fixVideoMetadataPerms(file_path, type) {
     if (is_windows) return;
-    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
-                                                   : config_api.getConfigItem('ytdl_video_folder_path');
 
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
+    const file_path_no_extension = removeFileExtension(file_path);
+
     const files_to_fix = [
         // JSONs
-        path.join(customPath, name + '.info.json'),
+        file_path_no_extension + '.info.json',
-        path.join(customPath, name + ext + '.info.json'),
+        file_path_no_extension + ext + '.info.json',
         // Thumbnails
-        path.join(customPath, name + '.webp'),
+        file_path_no_extension + '.webp',
-        path.join(customPath, name + '.jpg')
+        file_path_no_extension + '.jpg'
     ];
 
     for (const file of files_to_fix) {
@@ -152,18 +203,68 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
     }
 }
 
-function deleteJSONFile(name, type, customPath = null) {
+function deleteJSONFile(file_path, type) {
-    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
-                                                   : config_api.getConfigItem('ytdl_video_folder_path');
-
     const ext = type === 'audio' ? '.mp3' : '.mp4';
-    let json_path = path.join(customPath, name + '.info.json');
+
-    let alternate_json_path = path.join(customPath, name + ext + '.info.json');
+    const file_path_no_extension = removeFileExtension(file_path);
+    
+    let json_path = file_path_no_extension + '.info.json';
+    let alternate_json_path = file_path_no_extension + ext + '.info.json';
 
     if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
+async function removeIDFromArchive(archive_path, id) {
+    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
+    if (!data) {
+        logger.error('Archive could not be found.');
+        return;
+    }
+
+    let dataArray = data.split('\n'); // convert file data in an array
+    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
+    let lastIndex = -1; // let say, we have not found the keyword
+
+    for (let index=0; index<dataArray.length; index++) {
+        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+            lastIndex = index; // found a line includes a id keyword
+            break;
+        }
+    }
+
+    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
+
+    // UPDATE FILE WITH NEW DATA
+    const updatedData = dataArray.join('\n');
+    await fs.writeFile(archive_path, updatedData);
+    if (line) return line;
+}
+
+function durationStringToNumber(dur_str) {
+    if (typeof dur_str === 'number') return dur_str;
+    let num_sum = 0;
+    const dur_str_parts = dur_str.split(':');
+    for (let i = dur_str_parts.length-1; i >= 0; i--) {
+      num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
+    }
+    return num_sum;
+}
+
+function getMatchingCategoryFiles(category, files) {
+    return files && files.filter(file => file.category && file.category.uid === category.uid);
+}
+
+function addUIDsToCategory(category, files) {
+    const files_that_match = getMatchingCategoryFiles(category, files);
+    category['uids'] = files_that_match.map(file => file.uid);
+    return files_that_match;
+}
+
+function getCurrentDownloader() {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    return details_json['downloader'];
+}
 
 async function recFindByExt(base,ext,files,result)
 {
@@ -193,6 +294,67 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
+function formatDateString(date_string) {
+    return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
+}
+
+function createEdgeNGrams(str) {
+    if (str && str.length > 3) {
+        const minGram = 3
+        const maxGram = str.length
+        
+        return str.split(" ").reduce((ngrams, token) => {
+            if (token.length > minGram) {   
+                for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
+                    ngrams = [...ngrams, token.substr(0, i)]
+                }
+            } else {
+                ngrams = [...ngrams, token]
+            }
+            return ngrams
+        }, []).join(" ")
+    }
+    
+    return str
+}
+
+// ffmpeg helper functions
+
+async function cropFile(file_path, start, end, ext) {
+    return new Promise(resolve => {
+        const temp_file_path = `${file_path}.cropped${ext}`;
+        let base_ffmpeg_call = ffmpeg(file_path);
+        if (start) {
+            base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
+        }
+        if (end) {
+            base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
+        }
+        base_ffmpeg_call
+            .on('end', () => {
+                logger.verbose(`Cropping for '${file_path}' complete.`);
+                fs.unlinkSync(file_path);
+                fs.moveSync(temp_file_path, file_path);
+                resolve(true);
+            })
+            .on('error', (err) => {
+                logger.error(`Failed to crop ${file_path}.`);
+                logger.error(err);
+                resolve(false);
+            }).save(temp_file_path);
+    });
+}
+
+/**
+ * setTimeout, but its a promise.
+ * @param {number} ms
+ */
+ async function wait(ms) {
+    await new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -215,13 +377,24 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
 module.exports = {
     getJSONMp3: getJSONMp3,
     getJSONMp4: getJSONMp4,
+    getJSON: getJSON,
     getTrueFileName: getTrueFileName,
     getDownloadedThumbnail: getDownloadedThumbnail,
     getExpectedFileSize: getExpectedFileSize,
     fixVideoMetadataPerms: fixVideoMetadataPerms,
     deleteJSONFile: deleteJSONFile,
+    removeIDFromArchive: removeIDFromArchive,
     getDownloadedFilesByType: getDownloadedFilesByType,
+    createContainerZipFile: createContainerZipFile,
+    durationStringToNumber: durationStringToNumber,
+    getMatchingCategoryFiles: getMatchingCategoryFiles,
+    addUIDsToCategory: addUIDsToCategory,
+    getCurrentDownloader: getCurrentDownloader,
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
+    formatDateString: formatDateString,
+    cropFile: cropFile,
+    createEdgeNGrams: createEdgeNGrams,
+    wait: wait,
     File: File
 }

chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: youtubedl-material
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "4.2"

chart/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "youtubedl-material.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "youtubedl-material.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "youtubedl-material.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "youtubedl-material.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

chart/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "youtubedl-material.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "youtubedl-material.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "youtubedl-material.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "youtubedl-material.labels" -}}
+helm.sh/chart: {{ include "youtubedl-material.chart" . }}
+{{ include "youtubedl-material.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "youtubedl-material.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "youtubedl-material.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "youtubedl-material.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "youtubedl-material.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

chart/templates/appdata-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.appdata.enabled (not .Values.persistence.appdata.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-appdata
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.appdata.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.appdata.size | quote }}
+  {{- if .Values.persistence.appdata.storageClass }}
+  {{- if (eq "-" .Values.persistence.appdata.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.appdata.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/audio-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.audio.enabled (not .Values.persistence.audio.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-audio
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.audio.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.audio.size | quote }}
+  {{- if .Values.persistence.audio.storageClass }}
+  {{- if (eq "-" .Values.persistence.audio.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.audio.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/deployment.yaml

@@ -0,0 +1,121 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "youtubedl-material.fullname" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "youtubedl-material.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "youtubedl-material.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "youtubedl-material.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 17442
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /app/appdata
+              name: appdata
+              {{- if .Values.persistence.appdata.subPath }}
+              subPath: {{ .Values.persistence.appdata.subPath }}
+              {{- end }}
+            - mountPath: /app/audio
+              name: audio
+              {{- if .Values.persistence.audio.subPath }}
+              subPath: {{ .Values.persistence.audio.subPath }}
+              {{- end }}
+            - mountPath: /app/video
+              name: video
+              {{- if .Values.persistence.video.subPath }}
+              subPath: {{ .Values.persistence.video.subPath }}
+              {{- end }}
+            - mountPath: /app/subscriptions
+              name: subscriptions
+              {{- if .Values.persistence.subscriptions.subPath }}
+              subPath: {{ .Values.persistence.subscriptions.subPath }}
+              {{- end }}
+            - mountPath: /app/users
+              name: users
+              {{- if .Values.persistence.users.subPath }}
+              subPath: {{ .Values.persistence.users.subPath }}
+              {{- end }}
+      volumes:
+        - name: appdata
+          {{- if .Values.persistence.appdata.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.appdata.existingClaim }}{{ .Values.persistence.appdata.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-appdata{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: audio
+          {{- if .Values.persistence.audio.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.audio.existingClaim }}{{ .Values.persistence.audio.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-audio{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: subscriptions
+          {{- if .Values.persistence.subscriptions.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.subscriptions.existingClaim }}{{ .Values.persistence.subscriptions.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-subscriptions{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: users
+          {{- if .Values.persistence.users.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.users.existingClaim }}{{ .Values.persistence.users.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-users{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: video
+          {{- if .Values.persistence.video.enabled}}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.persistence.video.existingClaim }}{{ .Values.persistence.video.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-video{{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

chart/templates/ingress.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "youtubedl-material.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+          {{- end }}
+    {{- end }}
+  {{- end }}

chart/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "youtubedl-material.fullname" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "youtubedl-material.selectorLabels" . | nindent 4 }}

chart/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "youtubedl-material.serviceAccountName" . }}
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/templates/subscriptions-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.subscriptions.enabled (not .Values.persistence.subscriptions.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-subscriptions
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.subscriptions.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.subscriptions.size | quote }}
+  {{- if .Values.persistence.subscriptions.storageClass }}
+  {{- if (eq "-" .Values.persistence.subscriptions.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.subscriptions.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "youtubedl-material.fullname" . }}-test-connection"
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "youtubedl-material.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

chart/templates/users-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.users.enabled (not .Values.persistence.users.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-users
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.users.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.users.size | quote }}
+  {{- if .Values.persistence.users.storageClass }}
+  {{- if (eq "-" .Values.persistence.users.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.users.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/templates/video-pvc.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.video.enabled (not .Values.persistence.video.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ template "youtubedl-material.fullname" . }}-video
+  labels:
+    {{- include "youtubedl-material.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.video.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.video.size | quote }}
+  {{- if .Values.persistence.video.storageClass }}
+  {{- if (eq "-" .Values.persistence.video.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.video.storageClass }}"
+  {{- end }}
+  {{- end }}
+  {{- end -}}

chart/values.yaml

@@ -0,0 +1,153 @@
+# Default values for youtubedl-material.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: tzahi12345/youtubedl-material
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 17442
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+persistence:
+  appdata:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 1Gi
+  audio:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  video:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  subscriptions:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+  users:
+    enabled: true
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ##
+    ## If you want to reuse an existing claim, you can pass the name of the PVC using
+    ## the existingClaim variable
+    # existingClaim: your-claim
+    # subPath: some-subpath
+    accessMode: ReadWriteOnce
+    size: 50Gi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

docker-compose.yml

@@ -3,6 +3,9 @@ services:
     ytdl_material:
         environment: 
             ALLOW_CONFIG_MUTATIONS: 'true'
+            ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
+            ytdl_use_local_db: 'false'
+            write_ytdl_config: 'true'
         restart: always
         volumes:
             - ./appdata:/app/appdata
@@ -12,4 +15,13 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:latest
+        image: tzahi12345/youtubedl-material:nightly
+    ytdl-mongo-db:
+        image: mongo
+        ports:
+            - "27017:27017"
+        logging:
+            driver: "none"          
+        container_name: mongo-db
+        volumes:
+            - ./db/:/data/db

package.json

@@ -5,12 +5,14 @@
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
-    "build": "ng build",
+    "build": "ng build --prod",
+    "prebuild": "node src/postbuild.mjs",
     "heroku-postbuild": "npm install --prefix backend",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e",
-    "electron": "ng build --base-href ./ && electron ."
+    "electron": "ng build --base-href ./ && electron .",
+    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
   },
   "engines": {
     "node": "12.3.1",
@@ -33,9 +35,12 @@
     "@ngneat/content-loader": "^5.0.0",
     "@videogular/ngx-videogular": "^2.1.0",
     "core-js": "^2.4.1",
+    "crypto-js": "^4.1.1",
     "file-saver": "^2.0.2",
     "filesize": "^6.1.0",
     "fingerprintjs2": "^2.1.0",
+    "fs-extra": "^10.0.0",
+    "material-icons": "^0.5.4",
     "nan": "^2.14.1",
     "ng-lazyload-image": "^7.0.1",
     "ngx-avatar": "^4.0.0",
@@ -45,6 +50,7 @@
     "tslib": "^2.0.0",
     "typescript": "~4.0.5",
     "web-animations-js": "^2.3.2",
+    "xliff-to-json": "^1.0.4",
     "zone.js": "~0.10.2"
   },
   "devDependencies": {
@@ -56,16 +62,20 @@
     "@types/file-saver": "^2.0.1",
     "@types/jasmine": "~3.6.0",
     "@types/node": "^12.11.1",
+    "@typescript-eslint/eslint-plugin": "^4.29.0",
+    "@typescript-eslint/parser": "^4.29.0",
     "codelyzer": "^6.0.0",
-    "electron": "^8.0.1",
+    "electron": "^13.6.6",
+    "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~5.0.0",
+    "karma": "~6.3.16",
     "karma-chrome-launcher": "~3.1.0",
     "karma-cli": "~1.0.1",
     "karma-coverage-istanbul-reporter": "~3.0.2",
     "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
+    "openapi-typescript-codegen": "^0.4.11",
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"

src/api-types/index.ts

@@ -0,0 +1,101 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
+export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
+export type { body_19 } from './models/body_19';
+export type { body_20 } from './models/body_20';
+export type { Category } from './models/Category';
+export { CategoryRule } from './models/CategoryRule';
+export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermissionsRequest';
+export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
+export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
+export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
+export type { ConcurrentStream } from './models/ConcurrentStream';
+export type { Config } from './models/Config';
+export type { ConfigResponse } from './models/ConfigResponse';
+export type { CreateCategoryRequest } from './models/CreateCategoryRequest';
+export type { CreateCategoryResponse } from './models/CreateCategoryResponse';
+export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
+export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
+export type { CropFileSettings } from './models/CropFileSettings';
+export type { DatabaseFile } from './models/DatabaseFile';
+export type { DBInfoResponse } from './models/DBInfoResponse';
+export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
+export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
+export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
+export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
+export type { DeleteUserRequest } from './models/DeleteUserRequest';
+export type { Download } from './models/Download';
+export type { DownloadArchiveRequest } from './models/DownloadArchiveRequest';
+export type { DownloadFileRequest } from './models/DownloadFileRequest';
+export type { DownloadRequest } from './models/DownloadRequest';
+export type { DownloadResponse } from './models/DownloadResponse';
+export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
+export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
+export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
+export type { File } from './models/File';
+export { FileType } from './models/FileType';
+export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
+export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
+export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
+export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
+export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
+export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
+export type { GetDownloadRequest } from './models/GetDownloadRequest';
+export type { GetDownloadResponse } from './models/GetDownloadResponse';
+export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
+export type { GetFileFormatsResponse } from './models/GetFileFormatsResponse';
+export type { GetFileRequest } from './models/GetFileRequest';
+export type { GetFileResponse } from './models/GetFileResponse';
+export type { GetFullTwitchChatRequest } from './models/GetFullTwitchChatRequest';
+export type { GetFullTwitchChatResponse } from './models/GetFullTwitchChatResponse';
+export type { GetLogsRequest } from './models/GetLogsRequest';
+export type { GetLogsResponse } from './models/GetLogsResponse';
+export type { GetMp3sResponse } from './models/GetMp3sResponse';
+export type { GetMp4sResponse } from './models/GetMp4sResponse';
+export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
+export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
+export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
+export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
+export type { GetRolesResponse } from './models/GetRolesResponse';
+export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
+export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
+export type { GetUsersResponse } from './models/GetUsersResponse';
+export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
+export type { inline_response_200_15 } from './models/inline_response_200_15';
+export type { LoginRequest } from './models/LoginRequest';
+export type { LoginResponse } from './models/LoginResponse';
+export type { Playlist } from './models/Playlist';
+export type { RegisterRequest } from './models/RegisterRequest';
+export type { RegisterResponse } from './models/RegisterResponse';
+export type { SetConfigRequest } from './models/SetConfigRequest';
+export type { SharingToggle } from './models/SharingToggle';
+export type { SubscribeRequest } from './models/SubscribeRequest';
+export type { SubscribeResponse } from './models/SubscribeResponse';
+export type { Subscription } from './models/Subscription';
+export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
+export type { SuccessObject } from './models/SuccessObject';
+export type { TableInfo } from './models/TableInfo';
+export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
+export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
+export type { TransferDBRequest } from './models/TransferDBRequest';
+export type { TransferDBResponse } from './models/TransferDBResponse';
+export type { TwitchChatMessage } from './models/TwitchChatMessage';
+export type { UnsubscribeRequest } from './models/UnsubscribeRequest';
+export type { UnsubscribeResponse } from './models/UnsubscribeResponse';
+export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
+export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
+export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
+export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
+export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
+export type { UpdaterStatus } from './models/UpdaterStatus';
+export type { UpdateServerRequest } from './models/UpdateServerRequest';
+export type { UpdateUserRequest } from './models/UpdateUserRequest';
+export type { User } from './models/User';
+export { UserPermission } from './models/UserPermission';
+export type { Version } from './models/Version';
+export type { VersionInfoResponse } from './models/VersionInfoResponse';
+export { YesNo } from './models/YesNo';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface AddFileToPlaylistRequest {
+    file_uid: string;
+    playlist_id: string;
+}

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { UserPermission } from './UserPermission';
+import { YesNo } from './YesNo';
+
+export interface BaseChangePermissionsRequest {
+    permission: UserPermission;
+    new_value: YesNo;
+}

src/api-types/models/Category.ts

@@ -0,0 +1,15 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { CategoryRule } from './CategoryRule';
+
+export interface Category {
+    name?: string;
+    uid?: string;
+    rules?: Array<CategoryRule>;
+    /**
+     * Overrides file output for downloaded files in category
+     */
+    custom_output?: string;
+}

src/api-types/models/CategoryRule.ts

@@ -0,0 +1,26 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface CategoryRule {
+    preceding_operator?: CategoryRule.preceding_operator;
+    comparator?: CategoryRule.comparator;
+}
+
+export namespace CategoryRule {
+
+    export enum preceding_operator {
+        OR = 'or',
+        AND = 'and',
+    }
+
+    export enum comparator {
+        INCLUDES = 'includes',
+        NOT_INCLUDES = 'not_includes',
+        EQUALS = 'equals',
+        NOT_EQUALS = 'not_equals',
+    }
+
+
+}

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+
+export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
+    role: string;
+}

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+
+export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
+    user_uid: string;
+}

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface CheckConcurrentStreamRequest {
+    /**
+     * UID of the concurrent stream
+     */
+    uid: string;
+}

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { ConcurrentStream } from './ConcurrentStream';
+
+export interface CheckConcurrentStreamResponse {
+    stream: ConcurrentStream;
+}

src/api-types/models/ConcurrentStream.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface ConcurrentStream {
+    playback_timestamp?: number;
+    unix_timestamp?: number;
+    playing?: boolean;
+}

src/api-types/models/Config.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface Config {
+    YoutubeDLMaterial: any;
+}

src/api-types/models/ConfigResponse.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Config } from './Config';
+
+export interface ConfigResponse {
+    config_file: Config;
+    success: boolean;
+}

src/api-types/models/CreateCategoryRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface CreateCategoryRequest {
+    name: string;
+}

src/api-types/models/CreateCategoryResponse.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Category } from './Category';
+
+export interface CreateCategoryResponse {
+    new_category?: Category;
+    success?: boolean;
+}

src/api-types/models/CreatePlaylistRequest.ts

@@ -0,0 +1,12 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { FileType } from './FileType';
+
+export interface CreatePlaylistRequest {
+    playlistName: string;
+    uids: Array<string>;
+    type: FileType;
+    thumbnailURL: string;
+}

src/api-types/models/CreatePlaylistResponse.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Playlist } from './Playlist';
+
+export interface CreatePlaylistResponse {
+    new_playlist: Playlist;
+    success: boolean;
+}

src/api-types/models/CropFileSettings.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface CropFileSettings {
+    cropFileStart: number;
+    cropFileEnd: number;
+}

src/api-types/models/DBInfoResponse.ts

@@ -0,0 +1,18 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { TableInfo } from './TableInfo';
+
+export interface DBInfoResponse {
+    using_local_db?: boolean;
+    stats_by_table?: {
+files?: TableInfo,
+playlists?: TableInfo,
+categories?: TableInfo,
+subscriptions?: TableInfo,
+users?: TableInfo,
+roles?: TableInfo,
+download_queue?: TableInfo,
+};
+}

src/api-types/models/DatabaseFile.ts

@@ -0,0 +1,22 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DatabaseFile {
+    id: string;
+    title: string;
+    thumbnailURL: string;
+    isAudio: boolean;
+    /**
+     * In seconds
+     */
+    duration: number;
+    url: string;
+    uploader: string;
+    size: number;
+    path: string;
+    upload_date: string;
+    uid: string;
+    sharingEnabled?: boolean;
+}

src/api-types/models/DeleteCategoryRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DeleteCategoryRequest {
+    category_uid: string;
+}

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DeleteMp3Mp4Request {
+    uid: string;
+    blacklistMode?: boolean;
+}

src/api-types/models/DeletePlaylistRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { FileType } from './FileType';
+
+export interface DeletePlaylistRequest {
+    playlist_id: string;
+    type: FileType;
+}

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -0,0 +1,15 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { SubscriptionRequestData } from './SubscriptionRequestData';
+
+export interface DeleteSubscriptionFileRequest {
+    file: string;
+    file_uid?: string;
+    sub: SubscriptionRequestData;
+    /**
+     * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
+     */
+    deleteForever?: boolean;
+}

src/api-types/models/DeleteUserRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DeleteUserRequest {
+    uid: string;
+}

src/api-types/models/Dictionary.ts

@@ -0,0 +1,7 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type Dictionary<T> = {
+    [key: string]: T;
+}

src/api-types/models/Download.ts

@@ -0,0 +1,26 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface Download {
+    uid: string;
+    ui_uid?: string;
+    running: boolean;
+    finished: boolean;
+    paused: boolean;
+    finished_step: boolean;
+    url: string;
+    type: string;
+    title: string;
+    step_index: number;
+    percent_complete: number;
+    timestamp_start: number;
+    /**
+     * Error text, set if download fails.
+     */
+    error?: string | null;
+    user_uid?: string;
+    sub_id?: string;
+    sub_name?: string;
+}

src/api-types/models/DownloadArchiveRequest.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DownloadArchiveRequest {
+    sub: {
+archive_dir: string,
+};
+}

src/api-types/models/DownloadFileRequest.ts

@@ -0,0 +1,13 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DownloadFileRequest {
+    uid?: string;
+    uuid?: string;
+    sub_id?: string;
+    playlist_id?: string;
+    url?: string;
+    type?: string;
+}

src/api-types/models/DownloadRequest.ts

@@ -0,0 +1,44 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { CropFileSettings } from './CropFileSettings';
+import { FileType } from './FileType';
+
+export interface DownloadRequest {
+    url: string;
+    /**
+     * Video format code. Overrides other quality options.
+     */
+    customQualityConfiguration?: string;
+    /**
+     * Custom command-line arguments for youtube-dl. Overrides all other options, except url.
+     */
+    customArgs?: string;
+    /**
+     * Additional command-line arguments for youtube-dl. Added to whatever args would normally be used.
+     */
+    additionalArgs?: string;
+    /**
+     * Custom output filename template.
+     */
+    customOutput?: string;
+    /**
+     * Login with this account ID
+     */
+    youtubeUsername?: string;
+    /**
+     * Account password
+     */
+    youtubePassword?: string;
+    /**
+     * Height of the video, if known
+     */
+    selectedHeight?: string;
+    /**
+     * Specify ffmpeg/avconv audio quality
+     */
+    maxBitrate?: string;
+    type?: FileType;
+    cropFileSettings?: CropFileSettings;
+}

src/api-types/models/DownloadResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Download } from './Download';
+
+export interface DownloadResponse {
+    download?: Download;
+}

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -0,0 +1,23 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { FileType } from './FileType';
+import { Subscription } from './Subscription';
+
+export interface DownloadTwitchChatByVODIDRequest {
+    /**
+     * File ID
+     */
+    id: string;
+    /**
+     * ID of the VOD
+     */
+    vodId: string;
+    type: FileType;
+    /**
+     * User UID
+     */
+    uuid?: string;
+    sub?: Subscription;
+}

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { TwitchChatMessage } from './TwitchChatMessage';
+
+export interface DownloadTwitchChatByVODIDResponse {
+    chat: Array<TwitchChatMessage>;
+}

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface DownloadVideosForSubscriptionRequest {
+    subID: string;
+}

src/api-types/models/File.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface File {
+    id?: string;
+}

src/api-types/models/FileType.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export enum FileType {
+    AUDIO = 'audio',
+    VIDEO = 'video',
+}

src/api-types/models/GenerateArgsResponse.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GenerateArgsResponse {
+    args?: Array<string>;
+}

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GenerateNewApiKeyResponse {
+    new_api_key: string;
+}

src/api-types/models/GetAllCategoriesResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Category } from './Category';
+
+export interface GetAllCategoriesResponse {
+    categories: Array<Category>;
+}

src/api-types/models/GetAllDownloadsRequest.ts

@@ -0,0 +1,11 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetAllDownloadsRequest {
+    /**
+     * Filters downloads with the array
+     */
+    uids?: Array<string> | null;
+}

src/api-types/models/GetAllDownloadsResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Download } from './Download';
+
+export interface GetAllDownloadsResponse {
+    downloads?: Array<Download>;
+}

src/api-types/models/GetAllFilesResponse.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { DatabaseFile } from './DatabaseFile';
+import { Playlist } from './Playlist';
+
+export interface GetAllFilesResponse {
+    files: Array<DatabaseFile>;
+    /**
+     * All video playlists
+     */
+    playlists: Array<Playlist>;
+}

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Subscription } from './Subscription';
+
+export interface GetAllSubscriptionsResponse {
+    subscriptions: Array<Subscription>;
+}

src/api-types/models/GetDownloadRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetDownloadRequest {
+    download_uid: string;
+}

src/api-types/models/GetDownloadResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { Download } from './Download';
+
+export interface GetDownloadResponse {
+    download?: Download;
+}

src/api-types/models/GetFileFormatsRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetFileFormatsRequest {
+    url?: string;
+}

src/api-types/models/GetFileFormatsResponse.ts

@@ -0,0 +1,12 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { File } from './File';
+
+export interface GetFileFormatsResponse {
+    success: boolean;
+    result: {
+formats?: Array<any>,
+};
+}

src/api-types/models/GetFileRequest.ts

@@ -0,0 +1,17 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { FileType } from './FileType';
+
+export interface GetFileRequest {
+    /**
+     * Video UID
+     */
+    uid: string;
+    type?: FileType;
+    /**
+     * User UID
+     */
+    uuid?: string;
+}

src/api-types/models/GetFileResponse.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { DatabaseFile } from './DatabaseFile';
+
+export interface GetFileResponse {
+    success: boolean;
+    file?: DatabaseFile;
+}

src/api-types/models/GetFullTwitchChatRequest.ts

@@ -0,0 +1,22 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import { FileType } from './FileType';
+import { Subscription } from './Subscription';
+
+export interface GetFullTwitchChatRequest {
+    /**
+     * File ID
+     */
+    id: string;
+    type: FileType;
+    /**
+     * User UID
+     */
+    uuid?: string;
+    /**
+     * Subscription
+     */
+    sub?: Subscription;
+}

src/api-types/models/GetFullTwitchChatResponse.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetFullTwitchChatResponse {
+    success: boolean;
+    error?: string;
+}

src/api-types/models/GetLogsRequest.ts

@@ -0,0 +1,8 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetLogsRequest {
+    lines?: number;
+}

src/api-types/models/GetLogsResponse.ts

@@ -0,0 +1,12 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+
+export interface GetLogsResponse {
+    /**
+     * Number of lines to retrieve from the bottom
+     */
+    logs?: string;
+    success?: boolean;
+}