Updated Angular to v14

Add permissions for tasks manager

.github/dependabot.yaml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/backend/"
+    schedule:
+      interval: "daily"

.github/workflows/close-issue-if-noresponse.yml

@@ -0,0 +1,38 @@
+name: No Response
+
+# Both `issue_comment` and `scheduled` event types are required for this Action
+# to work properly.
+on:
+  issue_comment:
+    types: [created]
+  schedule:
+    # Schedule for five minutes after the hour, every hour
+    - cron: '5 * * * *'
+
+# By specifying the access of one of the scopes, all of those that are not
+# specified are set to 'none'.
+permissions:
+  issues: write
+
+jobs:
+  noResponse:
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'Tzahi12345/YoutubeDL-Material' }}
+    steps:
+      - uses: lee-dohm/no-response@v0.5.0
+        with:
+          token: ${{ github.token }}
+          # Comment to post when closing an Issue for lack of response. Set to `false` to disable
+          closeComment: >
+            This issue has been automatically closed because there has been no response
+            to our request for more information from the original author. With only the
+            information that is currently in the issue, we don't have enough information
+            to take action. Please reach out if you have or find the answers we need so
+            that we can investigate further. We will re-open this issue if you provide us
+            with the requested information with a comment under this issue.
+            Thank you for your understanding and for trying to help make this application
+            a better one!
+          # Number of days of inactivity before an issue is closed for lack of response.
+          daysUntilClose: 21
+          # Label requiring a response.
+          responseRequiredLabel: "💬 response-needed"

.github/workflows/docker-release.yml

@@ -6,19 +6,25 @@ on:
       tags:
         description: 'Docker tags'
         required: true
+  release:
+    types: [published]
 
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+
       - name: Set hash
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
       - name: Get current date
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
       - name: create-json
         id: create-json
         uses: jsdaniell/create-json@1.1.2
@@ -26,15 +32,49 @@ jobs:
           name: "version.json"
           json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
+
+      - name: Set image tag
+        id: tags
+        run: |
+          if [ "${{ github.event.inputs.tags }}" != "" ]; then
+            echo "::set-output name=tags::${{ github.event.inputs.tags }}"
+          elif [ ${{ github.event.action }} == "release" ]; then
+            echo "::set-output name=tags::${{ github.event.release.tag_name }}"
+          else
+            echo "Unknown workflow trigger: ${{ github.event.action }}! Cannot determine default tag."
+            exit 1
+          fi
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,value=${{ steps.tags.outputs.tags }}
+            type=raw,value=latest
+
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
+
       - name: setup multi-arch docker build
         uses: docker/setup-buildx-action@v1
+
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: build & push images
         uses: docker/build-push-action@v2
         with:
@@ -42,4 +82,5 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          tags: ${{ github.event.inputs.tags }}
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}

.github/workflows/docker.yml

@@ -13,6 +13,9 @@ on:
       - '**.pem'
       - '.dockerignore'
       - '.gitignore'
+  schedule:
+    - cron:  '34 4 * * 2'
+  workflow_dispatch:
 
 jobs:
   build-and-push:
@@ -20,12 +23,15 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+
       - name: Set hash
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
       - name: Get current date
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
       - name: create-json
         id: create-json
         uses: jsdaniell/create-json@1.1.2
@@ -33,15 +39,42 @@ jobs:
           name: "version.json"
           json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
+
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
+
       - name: setup multi-arch docker build
         uses: docker/setup-buildx-action@v1
+
+      - name: Generate Docker image metadata
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        # Defaults:
+        #     DOCKERHUB_USERNAME  : tzahi12345
+        #     DOCKERHUB_REPO      : youtubedl-material
+        #     DOCKERHUB_MASTER_TAG: nightly
+        with:
+          images: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}
+            ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }}
+          tags: |
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}-{{ date 'YYYY-MM-DD' }}
+            type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}
+            type=sha,prefix=sha-,format=short
+
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: build & push images
         uses: docker/build-push-action@v2
         with:
@@ -49,8 +82,5 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          # Defaults:
+          tags: ${{ steps.docker-meta.outputs.tags }}
-          #     DOCKERHUB_USERNAME  : tzahi12345
+          labels: ${{ steps.docker-meta.outputs.labels }}
-          #     DOCKERHUB_REPO      : youtubedl-material
-          #     DOCKERHUB_MASTER_TAG: nightly
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}:${{secrets.DOCKERHUB_MASTER_TAG}}

Dockerfile

@@ -1,75 +1,69 @@
+# Fetching our ffmpeg
 FROM ubuntu:22.04 AS ffmpeg
-
 ENV DEBIAN_FRONTEND=noninteractive
+# Use script due local build compability
+COPY ffmpeg-fetch.sh .
+RUN sh ./ffmpeg-fetch.sh
 
-COPY docker-build.sh .
-RUN sh ./docker-build.sh
 
-#--------------# Stage 2
+# Create our Ubuntu 22.04 with node 16
+# Go to 20.04
+FROM ubuntu:20.04 AS base
+ARG DEBIAN_FRONTEND=noninteractive
+ENV UID=1000
+ENV GID=1000
+ENV USER=youtube
+ENV NO_UPDATE_NOTIFIER=true
+ENV PM2_HOME=/app/pm2
+ENV ALLOW_CONFIG_MUTATIONS=true
+RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
+    apt update && \
+    apt install -y --no-install-recommends curl ca-certificates tzdata && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt install -y --no-install-recommends nodejs && \
+    npm -g install npm && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
 
-FROM ubuntu:22.04 as frontend
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install \
-  curl \
-  gnupg \
-  # Ubuntu 22.04 ships Node.JS 12 by default :)
-  nodejs \
-  # needed on 21.10 and before, maybe not on 22.04 YARN: brings along npm, solves dependency conflicts,
-  # spares us this spaghetti approach: https://stackoverflow.com/a/60547197
-  npm && \
-  apt-get install -f && \
-  npm config set strict-ssl false && \
-  npm install -g @angular/cli
 
+# Build frontend
+FROM base as frontend
+RUN npm install -g @angular/cli
 WORKDIR /build
-COPY [ "package.json", "package-lock.json", "/build/" ]
+COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
-RUN npm install
-
-COPY [ "angular.json", "tsconfig.json", "/build/" ]
 COPY [ "src/", "/build/src/" ]
-RUN npm run build
+RUN npm install && \
+    npm run build && \
+    ls -al /build/backend/public
 
-#--------------# Final Stage
-
-FROM ubuntu:22.04
-
-ENV UID=1000 \
-  GID=1000 \
-  USER=youtube \
-  NO_UPDATE_NOTIFIER=true
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
-
-RUN apt-get update && apt-get -y install \
-  npm \
-  python2 \
-  python3 \
-  gosu \
-  atomicparsley && \
-  apt-get install -f && \
-  apt-get autoremove --purge && \
-  apt-get autoremove && \
-  apt-get clean && \
-  rm -rf /var/lib/apt
 
+# Install backend deps
+FROM base as backend
 WORKDIR /app
+COPY [ "backend/","/app/" ]
+RUN npm config set strict-ssl false && \
+    npm install --prod && \
+    ls -al
+
+
+# Final image
+FROM base
+RUN npm install -g pm2 && \
+    apt update && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install tcd
+WORKDIR /app
+# User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
-COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
+COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
-ENV PM2_HOME=/app/pm2
-RUN npm config set strict-ssl false && \
-  npm install pm2 -g && \
-  npm install && chown -R $UID:$GID ./
-
-# needed for ubuntu, see #596
-RUN ln -s /usr/bin/python3 /usr/bin/python
-
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
+RUN chmod +x /app/fix-scripts/*.sh
+# Add some persistence data
+#VOLUME ["/app/appdata"]
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime", "pm2.config.js" ]
+CMD [ "npm","start" ]

Dockerfile.heroku

@@ -1,2 +1,2 @@
-FROM tzahi12345/youtubedl-material:nightly
+FROM tzahi12345/youtubedl-material:latest
-CMD [ "pm2-runtime", "pm2.config.js" ]
+CMD [ "npm", "start" ]

Public API v1.yaml

@@ -97,6 +97,11 @@ paths:
       summary: Get all files
       description: Gets all files and playlists stored in the db
       operationId: get-getAllFiles
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GetAllFilesRequest'
       responses:
         '200':
           description: OK
@@ -129,6 +134,27 @@ paths:
           description: User is not authorized to view the file.
       security:
         - Auth query parameter: []
+  /api/updateFile:
+    post:
+      tags:
+        - files
+      summary: Updates file database object
+      description: Updates a file db object using its uid and a change object.
+      operationId: post-updateFile
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UpdateFileRequest'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessObject'
+      security:
+        - Auth query parameter: []
   /api/enableSharing:
     post:
       tags:
@@ -841,17 +867,10 @@ paths:
         - Auth query parameter: []
       tags:
         - downloader
-  /api/clearFinishedDownloads:
+  /api/clearDownloads:
     post:
-      tags:
+      summary: Clear multiple downloads
-        - downloader
+      operationId: post-api-clear-downloads
-      summary: Clear finished downloads
-      operationId: post-api-clear-finished-downloads
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
       responses:
         '200':
           description: OK
@@ -859,8 +878,17 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/SuccessObject'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ClearDownloadsRequest'
+        description: ''
+      description: "Clears multiple downloads based on a given filter."
       security:
         - Auth query parameter: []
+      tags:
+        - downloader
   /api/getTask:
     post:
       summary: Get info for one task
@@ -1507,6 +1535,8 @@ components:
       properties:
         success:
           type: boolean
+        error:
+          type: string
     FileType:
       type: string
       enum:
@@ -1607,6 +1637,15 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Download'
+    ClearDownloadsRequest:
+      type: object
+      properties:
+        clear_finished:
+          type: boolean
+        clear_paused:
+          type: boolean
+        clear_errors:
+          type: boolean
     GetTaskRequest:
       type: object
       properties:
@@ -1690,6 +1729,41 @@ components:
           description: All video playlists
           items:
             $ref: '#/components/schemas/Playlist'
+    GetAllFilesRequest:
+      type: object
+      properties:
+        sort:
+          $ref: '#/components/schemas/Sort'
+        range:
+          type: array
+          items:
+            type: number
+            description: Two elements allowed, start index and end index
+          minItems: 2
+          maxItems: 2
+        text_search:
+          type: string
+          description: Filter files by title
+        file_type_filter:
+          $ref: '#/components/schemas/FileTypeFilter'
+        sub_id:
+          type: string
+          description: Include if you want to filter by subscription
+    Sort:
+      type: object
+      properties:
+        by:
+          type: string
+          description: Property to sort by
+        order:
+          type: number
+          description: 1 for ascending, -1 for descending
+    FileTypeFilter:
+      type: string
+      enum:
+        - audio_only
+        - video_only
+        - both
     GetAllFilesResponse:
       required:
         - files
@@ -1727,6 +1801,18 @@ components:
           type: boolean
         file:
           $ref: '#/components/schemas/DatabaseFile'
+    UpdateFileRequest:
+      required:
+        - uid
+        - change_obj
+      type: object
+      properties:
+        uid:
+          type: string
+          description: Video UID
+        change_obj:
+          type: object
+          description: Object with fields to update as keys and their new values
     SharingToggle:
       required:
         - uid
@@ -1740,7 +1826,6 @@ components:
       required:
         - name
         - url
-        - streamingOnly
       type: object
       properties:
         name:
@@ -1853,7 +1938,6 @@ components:
         - uids
         - playlistName
         - thumbnailURL
-        - type
       type: object
       properties:
         playlistName:
@@ -1862,8 +1946,6 @@ components:
           type: array
           items:
             type: string
-        type:
-          $ref: '#/components/schemas/FileType'
         thumbnailURL:
           type: string
     CreatePlaylistResponse:
@@ -1893,15 +1975,17 @@ components:
       required:
         - playlist
         - success
-        - type
       type: object
       properties:
         playlist:
           $ref: '#/components/schemas/Playlist'
-        type:
-          $ref: '#/components/schemas/FileType'
         success:
           type: boolean
+        file_objs:
+          type: array
+          description: File objects for every uid in the playlist's uids property, in the same order
+          items:
+            $ref: '#/components/schemas/DatabaseFile'
     GetPlaylistsRequest:
       type: object
       properties:
@@ -1926,13 +2010,10 @@ components:
     DeletePlaylistRequest:
       required:
         - playlist_id
-        - type
       type: object
       properties:
         playlist_id:
           type: string
-        type:
-          $ref: '#/components/schemas/FileType'
     DownloadFileRequest:
       type: object
       properties:
@@ -2153,7 +2234,6 @@ components:
           type: boolean
         result:
           allOf:
-          - $ref: '#/components/schemas/file'
           - type: object
             properties:
               formats:
@@ -2311,6 +2391,9 @@ components:
           type: string
         thumbnailURL:
           type: string
+          description: Backup if thumbnailPath is not defined
+        thumbnailPath:
+          type: string
         isAudio:
           type: boolean
         duration:
@@ -2322,6 +2405,7 @@ components:
           type: string
         size:
           type: number
+          description: In bytes
         path:
           type: string
         upload_date:
@@ -2330,6 +2414,22 @@ components:
           type: string
         sharingEnabled:
           type: boolean
+        category:
+          $ref: '#/components/schemas/Category'
+        view_count:
+          type: number
+        local_view_count:
+          type: number
+        sub_id:
+          type: string
+        registered:
+          type: number
+        height:
+          type: number
+          description: In pixels, only for videos
+        abr:
+          type: number
+          description: In Kbps
     Playlist:
       required:
         - uids
@@ -2359,6 +2459,8 @@ components:
           type: number
         user_uid:
           type: string
+        auto:
+          type: boolean
     Download:
       required:
         - url
@@ -2409,6 +2511,8 @@ components:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key
@@ -2423,6 +2527,8 @@ components:
       properties:
         key:
           type: string
+        title:
+          type: string
         last_ran:
           type: number
         last_confirmed:
@@ -2503,7 +2609,6 @@ components:
         - url
         - type
         - user_uid
-        - streamingOnly
         - isPlaylist
         - videos
       type: object
@@ -2519,8 +2624,6 @@ components:
         user_uid:
           type: string
           nullable: true
-        streamingOnly:
-          type: boolean
         isPlaylist:
           type: boolean
         archive:
@@ -2545,28 +2648,6 @@ components:
           type: string
         passhash:
           type: string
-        files:
-          type: object
-          properties:
-            audio:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-            video:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-        playlists:
-          type: object
-          properties:
-            audio:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
-            video:
-              type: array
-              items:
-                $ref: '#/components/schemas/file'
         subscriptions:
           type: array
           items:

README.md

@@ -12,16 +12,6 @@ Now with [Docker](#Docker) support!
 
 <hr>
 
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
 Check out the prerequisites, and go to the installation section. Easy as pie!
@@ -58,6 +48,7 @@ sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 Optional dependencies:
 
 * AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
@@ -91,7 +82,7 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ### Host-specific instructions
 
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
 
 ### Setup
 
@@ -102,8 +93,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for
 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
 
-NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
-
 ### Custom UID/GID
 
 By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
@@ -114,6 +103,12 @@ environment:
     GID: YOUR_GID
 ```
 
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
 ## API
 
 [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)

SECURITY.md

@@ -2,16 +2,16 @@
 
 ## Supported Versions
 
-Currently all work on this project goes into the nightly builds.
+If you would like to see the latest updates, use the `nightly` tag on Docker.
-4.2's RELEASE build is now quite old and should be considered legacy.
-We urge users to use the nightly releases, because the project
-constantly sees fixes.
 
-| Version       | Supported          |
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
-| ------------- | ------------------ |
+
-| 4.2 Nightlies | :white_check_mark: |
+| Version              | Supported          |
-| 4.2 Release   | :x:                |
+| -------------------- | ------------------ |
-| < 4.2         | :x:                |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
 
 ## Reporting a Vulnerability
 

angular.json

@@ -30,7 +30,8 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "scripts": [],
             "vendorChunk": true,
@@ -118,7 +119,8 @@
               "src/backend"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "scripts": []
           },
@@ -151,7 +153,8 @@
             "tsConfig": "src/tsconfig.spec.json",
             "scripts": [],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "src/bootstrap.min.css"
             ],
             "assets": [
               "src/assets",
@@ -179,7 +182,6 @@
       }
     }
   },
-  "defaultProject": "youtube-dl-material",
   "schematics": {
     "@schematics/angular:component": {
       "prefix": "app",

backend/app.js

@@ -68,7 +68,8 @@ db.defaults(
         configWriteFlag: false,
         downloads: {},
         subscriptions: [],
-        files_to_db_migration_complete: false
+        files_to_db_migration_complete: false,
+        tasks_manager_role_migration_complete: false
 }).write();
 
 users_db.defaults(
@@ -101,7 +102,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -148,16 +148,11 @@ if (fs.existsSync('version.json')) {
 
 // don't overwrite config if it already happened.. NOT
 // let alreadyWritten = db.get('configWriteFlag').value();
-let writeConfigMode = process.env.write_ytdl_config;
 
 // checks if config exists, if not, a config is auto generated
 config_api.configExistsCheck();
 
-if (writeConfigMode) {
+setAndLoadConfig();
-    setAndLoadConfig();
-} else {
-    loadConfig();
-}
 
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
@@ -188,13 +183,22 @@ async function checkMigrations() {
     if (!new_db_system_migration_complete) {
         logger.info('Beginning migration: 4.2->4.3+')
         let success = await db_api.importJSONToDB(db.value(), users_db.value());
-
+        await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first
         // sets migration to complete
         db.set('new_db_system_migration_complete', true).write();
         if (success) { logger.info('4.2->4.3+ migration complete!'); }
         else { logger.error('Migration failed: 4.2->4.3+'); }
     }
 
+    const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value();
+    if (!tasks_manager_role_migration_complete) {
+        logger.info('Checking if tasks manager role permissions exist for admin user...');
+        const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes');
+        if (success) logger.info('Task manager permissions check complete!');
+        else logger.error('Failed to auto add tasks manager permissions to admin role!');
+        db.set('tasks_manager_role_migration_complete', true).write();
+    }
+
     return true;
 }
 
@@ -484,8 +488,9 @@ async function setAndLoadConfig() {
 }
 
 async function setConfigFromEnv() {
-    let config_items = getEnvConfigItems();
+    const config_items = getEnvConfigItems();
-    let success = config_api.setConfigItems(config_items);
+    if (!config_items || config_items.length === 0) return true;
+    const success = config_api.setConfigItems(config_items);
     if (success) {
         logger.info('Config items set using ENV variables.');
         await utils.wait(100);
@@ -500,12 +505,13 @@ async function loadConfig() {
     loadConfigValues();
 
     // connect to DB
-    await db_api.connectToDB();
+    if (!config_api.getConfigItem('ytdl_use_local_db'))
+        await db_api.connectToDB();
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
     // creates archive path if missing
-    await fs.ensureDir(archivePath);
+    await fs.ensureDir(utils.getArchiveFolder());
 
     // check migrations
     await checkMigrations();
@@ -575,7 +581,11 @@ async function watchSubscriptions() {
 
     if (!subscriptions) return;
 
-    const valid_subscriptions = subscriptions.filter(sub => !sub.paused);
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
 
     let subscriptions_amount = valid_subscriptions.length;
     let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
@@ -912,11 +922,11 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
     let files = null;
-    let playlists = null;
+    const sort = req.body.sort;
-    let sort = req.body.sort;
+    const range = req.body.range;
-    let range = req.body.range;
+    const text_search = req.body.text_search;
-    let text_search = req.body.text_search;
+    const file_type_filter = req.body.file_type_filter;
-    let file_type_filter = req.body.file_type_filter;
+    const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     const filter_obj = {user_uid: uuid};
@@ -929,27 +939,42 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
         }
     }
 
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
     if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
     else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
     
     files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
-    let file_count = await db_api.getRecords('files', filter_obj, true);
+    const file_count = await db_api.getRecords('files', filter_obj, true);
-    playlists = await db_api.getRecords('playlists', {user_uid: uuid});
-
-    const categories = await categories_api.getCategoriesAsPlaylists(files);
-    if (categories) {
-        playlists = playlists.concat(categories);
-    }
 
     files = JSON.parse(JSON.stringify(files));
 
     res.send({
         files: files,
         file_count: file_count,
-        playlists: playlists
     });
 });
 
+app.post('/api/updateFile', optionalJwt, async function (req, res) {
+    const uid = req.body.uid;
+    const change_obj = req.body.change_obj;
+
+    const file = await db_api.updateRecord('files', {uid: uid}, change_obj);
+
+    if (!file) {
+        res.send({
+            success: false,
+            error: 'File could not be found'
+        });
+    } else {
+        res.send({
+            success: true
+        });
+    }
+});
+
 app.post('/api/checkConcurrentStream', async (req, res) => {
     const uid = req.body.uid;
 
@@ -1257,7 +1282,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
     subscription = JSON.parse(JSON.stringify(subscription));
 
     // get sub videos
-    if (subscription.name && !subscription.streamingOnly) {
+    if (subscription.name) {
         var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
         subscription['videos'] = parsed_files;
         // loop through files for extra processing
@@ -1267,19 +1292,6 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
         }
 
-        res.send({
-            subscription: subscription,
-            files: parsed_files
-        });
-    } else if (subscription.name && subscription.streamingOnly) {
-        // return list of videos
-        let parsed_files = [];
-        if (subscription.videos) {
-            for (let i = 0; i < subscription.videos.length; i++) {
-                const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
-            }
-        }
         res.send({
             subscription: subscription,
             files: parsed_files
@@ -1324,9 +1336,8 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
 app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
-    let type = req.body.type;
 
-    const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1354,7 +1365,6 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     res.send({
         playlist: playlist,
         file_objs: file_objs,
-        type: playlist && playlist.type,
         success: !!playlist
     });
 });
@@ -1365,7 +1375,7 @@ app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
 
     let playlists = await db_api.getRecords('playlists', {user_uid: uuid});
     if (include_categories) {
-        const categories = await categories_api.getCategoriesAsPlaylists(files);
+        const categories = await categories_api.getCategoriesAsPlaylists();
         if (categories) {
             playlists = playlists.concat(categories);
         }
@@ -1669,9 +1679,15 @@ app.post('/api/download', optionalJwt, async (req, res) => {
     }
 });
 
-app.post('/api/clearFinishedDownloads', optionalJwt, async (req, res) => {
+app.post('/api/clearDownloads', optionalJwt, async (req, res) => {
     const user_uid = req.isAuthenticated() ? req.user.uid : null;
-    const success = db_api.removeAllRecords('download_queue', {finished: true, user_uid: user_uid});
+    const clear_finished = req.body.clear_finished;
+    const clear_paused = req.body.clear_paused;
+    const clear_errors = req.body.clear_errors;
+    let success = true;
+    if (clear_finished) success &= await db_api.removeAllRecords('download_queue', {finished: true,        user_uid: user_uid});
+    if (clear_paused)   success &= await db_api.removeAllRecords('download_queue', {paused:   true,        user_uid: user_uid});
+    if (clear_errors)   success &= await db_api.removeAllRecords('download_queue', {error:    {$ne: null}, user_uid: user_uid});
     res.send({success: success});
 });
 

backend/appdata/default.json

@@ -31,7 +31,8 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
       "generate_NFO_files": false
@@ -63,7 +64,7 @@
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
-      "default_downloader": "youtube-dl",
+      "default_downloader": "yt-dlp",
       "use_default_downloading_agent": true,
       "custom_downloading_agent": "",
       "multi_user_mode": false,

backend/authentication/auth.js

@@ -171,8 +171,12 @@ exports.registerUser = async function(req, res) {
 
 
 exports.login = async (username, password) => {
+  // even if we're using LDAP, we still want users to be able to login using internal credentials
   const user = await db_api.getRecord('users', {name: username});
-  if (!user) { logger.error(`User ${username} not found`); return false }
+  if (!user) {
+    if (config_api.getConfigItem('ytdl_auth_method') === 'internal') logger.error(`User ${username} not found`);
+    return false;
+  }
   if (user.auth_method && user.auth_method !== 'internal') { return false }
   return await bcrypt.compare(password, user.passhash) ? user : false;
 }
@@ -357,7 +361,6 @@ exports.userHasPermission = async function(user_uid, permission) {
     logger.error('Invalid role ' + role);
     return false;
   }
-  const role_permissions = (await db_api.getRecords('roles'))['permissions'];
 
   const user_has_explicit_permission = user_obj['permissions'].includes(permission);
   const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -372,7 +375,8 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 
   // no overrides, let's check if the role has the permission
-  if (role_permissions.includes(permission)) {
+  const role_has_permission = await exports.roleHasPermissions(role, permission);
+  if (role_has_permission) {
     return true;
   } else {
     logger.verbose(`User ${user_uid} failed to get permission ${permission}`);
@@ -380,6 +384,16 @@ exports.userHasPermission = async function(user_uid, permission) {
   }
 }
 
+exports.roleHasPermissions = async function(role, permission) {
+  const role_obj = await db_api.getRecord('roles', {key: role})
+  if (!role) {
+    logger.error(`Role ${role} does not exist!`);
+  }
+  const role_permissions = role_obj['permissions'];
+  if (role_permissions && role_permissions.includes(permission)) return true;
+  else return false;
+}
+
 exports.userPermissions = async function(user_uid) {
   let user_permissions = [];
   const user_obj = await db_api.getRecord('users', ({uid: user_uid}));

backend/categories.js

@@ -55,17 +55,18 @@ async function getCategories() {
     return categories ? categories : null;
 }
 
-async function getCategoriesAsPlaylists(files = null) {
+async function getCategoriesAsPlaylists() {
     const categories_as_playlists = [];
     const available_categories = await getCategories();
-    if (available_categories && files) {
+    if (available_categories) {
         for (let category of available_categories) {
-            const files_that_match = utils.addUIDsToCategory(category, files);
+            const files_that_match = await db_api.getRecords('files', {'category.uid': category['uid']});
             if (files_that_match && files_that_match.length > 0) {
                 category['thumbnailURL'] = files_that_match[0].thumbnailURL;
                 category['thumbnailPath'] = files_that_match[0].thumbnailPath;
                 category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
                 category['id'] = category['uid'];
+                category['auto'] = true;
                 categories_as_playlists.push(category);
             }
         }

backend/config.js

@@ -127,7 +127,7 @@ function setConfigItem(key, value) {
     success = setConfigFile(config_json);
 
     return success;
-};
+}
 
 function setConfigItems(items) {
     let success = false;
@@ -206,7 +206,8 @@ const DEFAULT_CONFIG = {
         "use_youtube_API": false,
         "youtube_API_key": "",
         "use_twitch_API": false,
-        "twitch_API_key": "",
+        "twitch_client_ID": "",
+        "twitch_client_secret": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
         "generate_NFO_files": false
@@ -238,7 +239,7 @@ const DEFAULT_CONFIG = {
         "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
       },
       "Advanced": {
-        "default_downloader": "youtube-dl",
+        "default_downloader": "yt-dlp",
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",
         "multi_user_mode": false,

backend/consts.js

@@ -102,9 +102,13 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_use_twitch_api',
         'path': 'YoutubeDLMaterial.API.use_twitch_API'
     },
-    'ytdl_twitch_api_key': {
+    'ytdl_twitch_client_id': {
-        'key': 'ytdl_twitch_api_key',
+        'key': 'ytdl_twitch_client_id',
-        'path': 'YoutubeDLMaterial.API.twitch_API_key'
+        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
+    },
+    'ytdl_twitch_client_secret': {
+        'key': 'ytdl_twitch_client_secret',
+        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
     },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
@@ -217,7 +221,8 @@ exports.AVAILABLE_PERMISSIONS = [
     'subscriptions',
     'sharing',
     'advanced_download',
-    'downloads_manager'
+    'downloads_manager',
+    'tasks_manager'
 ];
 
 exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
@@ -301,4 +306,4 @@ const YTDL_ARGS_WITH_VALUES = [
 // we're using a Set here for performance
 exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
 
-exports.CURRENT_VERSION = 'v4.2';
+exports.CURRENT_VERSION = 'v4.3';

backend/db.js

@@ -198,7 +198,7 @@ async function registerFileDBManual(file_object) {
     path_object = path.parse(file_object['path']);
     file_object['path'] = path.format(path_object);
 
-    exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
+    await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
 
     return file_object;
 }
@@ -357,7 +357,7 @@ exports.addMetadataPropertyToDB = async (property_key) => {
     }
 }
 
-exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
+exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
     const first_video = await exports.getVideo(uids[0]);
     const thumbnailToUse = first_video['thumbnailURL'];
     
@@ -366,7 +366,6 @@ exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => {
         uids: uids,
         id: uuid(),
         thumbnailURL: thumbnailToUse,
-        type: type,
         registered: Date.now(),
         randomize_order: false
     };
@@ -387,9 +386,9 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
     if (!playlist) {
         playlist = await exports.getRecord('categories', {uid: playlist_id});
         if (playlist) {
-            // category found
+            const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
-            const files = await exports.getFiles(user_uid);
+            playlist['uids'] = uids;
-            utils.addUIDsToCategory(playlist, files);
+            playlist['auto'] = true;
         }
     }
 
@@ -495,8 +494,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+        const archive_path = utils.getArchiveFolder(type, uuid);
-        const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
 
         // get ID from JSON
 
@@ -504,14 +502,8 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
         let id = null;
         if (jsonobj) id = jsonobj.id;
 
-        // use subscriptions API to remove video from the archive file, and write it to the blacklist
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-        if (await fs.pathExists(archive_path)) {
+        await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
-            const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
-            if (blacklistMode && line) await writeToBlacklist(type, line);
-        } else {
-            logger.info('Could not find archive file for audio files. Creating...');
-            await fs.close(await fs.open(archive_path, 'w'));
-        }
     }
 
     if (jsonExists) await fs.unlink(jsonPath);
@@ -629,7 +621,7 @@ exports.bulkInsertRecordsIntoTable = async (table, docs) => {
 exports.getRecord = async (table, filter_obj) => {
     // local db override
     if (using_local_db) {
-        return applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
+        return exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').value();
     }
 
     return await database.collection(table).findOne(filter_obj);
@@ -638,7 +630,7 @@ exports.getRecord = async (table, filter_obj) => {
 exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
     // local db override
     if (using_local_db) {
-        let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        let cursor = filter_obj ? exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
         if (sort) {
             cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
         }
@@ -664,7 +656,7 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
 exports.updateRecord = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
         return true;
     }
 
@@ -677,7 +669,7 @@ exports.updateRecord = async (table, filter_obj, update_obj) => {
 exports.updateRecords = async (table, filter_obj, update_obj) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
         return true;
     }
 
@@ -722,7 +714,7 @@ exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
 exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).push(value).write();
         return true;
     }
 
@@ -733,7 +725,7 @@ exports.pushToRecordsArray = async (table, filter_obj, key, value) => {
 exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').get(key).pull(value).write();
         return true;
     }
 
@@ -746,7 +738,7 @@ exports.pullFromRecordsArray = async (table, filter_obj, key, value) => {
 exports.removeRecord = async (table, filter_obj) => {
     // local db override
     if (using_local_db) {
-        applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+        exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
         return true;
     }
 
@@ -757,7 +749,7 @@ exports.removeRecord = async (table, filter_obj) => {
 // exports.removeRecordsByUIDBulk = async (table, uids) => {
 //     // local db override
 //     if (using_local_db) {
-//         applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+//         exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
 //         return true;
 //     }
 
@@ -821,7 +813,7 @@ exports.removeAllRecords = async (table = null, filter_obj = null) => {
     if (using_local_db) {
         for (let i = 0; i < tables_to_remove.length; i++) {
             const table_to_remove = tables_to_remove[i];
-            if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
+            if (filter_obj) exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
             else local_db.assign({[table_to_remove]: []}).write();
             logger.debug(`Successfully removed records from ${table_to_remove}`);
         }
@@ -933,6 +925,7 @@ exports.importJSONToDB = async (db_json, users_json) => {
 const createFilesRecords = (files, subscriptions) => {
     for (let i = 0; i < subscriptions.length; i++) {
         const subscription = subscriptions[i];
+        if (!subscription['videos']) continue;
         subscription['videos'] = subscription['videos'].map(file => ({ ...file, sub_id: subscription['id'], user_uid: subscription['user_uid'] ? subscription['user_uid'] : undefined}));
         files = files.concat(subscriptions[i]['videos']);
     }
@@ -993,7 +986,7 @@ exports.backupDB = async () => {
     const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
     const path_to_backups = path.join(backup_dir, backup_file_name);
 
-    logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
+    logger.info(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
 
     const table_to_records = {};
     for (let i = 0; i < tables_list.length; i++) {
@@ -1040,10 +1033,11 @@ exports.transferDB = async (local_to_remote) => {
         table_to_records[table] = await exports.getRecords(table);
     }
 
+    logger.info('Backup up DB...');
+    await exports.backupDB(); // should backup always
+
     using_local_db = !local_to_remote;
     if (local_to_remote) {
-        logger.debug('Backup up DB...');
-        await exports.backupDB();
         const db_connected = await exports.connectToDB(5, true);
         if (!db_connected) {
             logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1075,8 +1069,13 @@ exports.transferDB = async (local_to_remote) => {
     This function is necessary to emulate mongodb's ability to search for null or missing values.
         A filter of null or undefined for a property will find docs that have that property missing, or have it
         null or undefined. We want that same functionality for the local DB as well
+
+        error:    {$ne: null}
+          ^            ^
+          |            |
+      filter_prop  filter_prop_value
 */
-const applyFilterLocalDB = (db_path, filter_obj, operation) => {
+exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     const filter_props = Object.keys(filter_obj);
     const return_val = db_path[operation](record => {
         if (!filter_props) return true;
@@ -1085,14 +1084,20 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
             const filter_prop = filter_props[i];
             const filter_prop_value = filter_obj[filter_prop];
             if (filter_prop_value === undefined || filter_prop_value === null) {
-                filtered &= record[filter_prop] === undefined || record[filter_prop] === null
+                filtered &= record[filter_prop] === undefined || record[filter_prop] === null;
             } else {
                 if (typeof filter_prop_value === 'object') {
-                    if (filter_prop_value['$regex']) {
+                    if ('$regex' in filter_prop_value) {
                         filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
+                    } else if ('$ne' in filter_prop_value) {
+                        filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
                     }
                 } else {
-                    filtered &= record[filter_prop] === filter_prop_value;
+                    // handle case of nested property check
+                    if (filter_prop.includes('.'))
+                        filtered &= utils.searchObjectByString(record, filter_prop) === filter_prop_value;
+                    else
+                        filtered &= record[filter_prop] === filter_prop_value;
                 }
             }
         }
@@ -1100,15 +1105,3 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
     });
     return return_val;
 }
-
-// archive helper functions
-
-async function writeToBlacklist(type, line) {
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}

backend/downloader.js

@@ -18,8 +18,6 @@ const db_api = require('./db');
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
 if (db_api.database_initialized) {
     setupDownloads();
 } else {
@@ -28,7 +26,7 @@ if (db_api.database_initialized) {
     });
 }
 
-exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
@@ -37,6 +35,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
             user_uid: user_uid,
             sub_id: sub_id,
             sub_name: sub_name,
+            prefetched_info: prefetched_info,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -108,6 +107,7 @@ exports.clearDownload = async (download_uid) => {
 }
 
 async function handleDownloadError(download_uid, error_message) {
+    if (!download_uid) return;
     await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
 }
 
@@ -186,7 +186,7 @@ async function collectInfo(download_uid) {
     let args = await exports.generateArgs(url, type, options, download['user_uid']);
 
     // get video info prior to download
-    let info = await getVideoInfoByURL(url, args, download_uid);
+    let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
 
     if (!info) {
         // info failed, error presumably already recorded
@@ -203,9 +203,12 @@ async function collectInfo(download_uid) {
         options.customOutput = category['custom_output'];
         options.noRelativePath = true;
         args = await exports.generateArgs(url, type, options, download['user_uid']);
-        info = await getVideoInfoByURL(url, args, download_uid);
+        args = utils.filterArgs(args, ['--no-simulate']);
+        info = await exports.getVideoInfoByURL(url, args, download_uid);
     }
 
+    download['category'] = category;
+
     // setup info required to calculate download progress
 
     const expected_file_size = utils.getExpectedFileSize(info);
@@ -226,7 +229,8 @@ async function collectInfo(download_uid) {
                                                                     options: options,
                                                                     files_to_check_for_progress: files_to_check_for_progress,
                                                                     expected_file_size: expected_file_size,
-                                                                    title: playlist_title ? playlist_title : info['title']
+                                                                    title: playlist_title ? playlist_title : info['title'],
+                                                                    prefetched_info: null
                                                                 });
 }
 
@@ -239,6 +243,7 @@ async function downloadQueuedFile(download_uid) {
     return new Promise(async resolve => {
         const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
         await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
 
         const url = download['url'];
@@ -246,9 +251,11 @@ async function downloadQueuedFile(download_uid) {
         const options = download['options'];
         const args = download['args'];
         const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
         if (options.customFileFolderPath) {
             fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
         }
         fs.ensureDirSync(fileFolderPath);
 
@@ -350,7 +357,7 @@ async function downloadQueuedFile(download_uid) {
                 if (file_objs.length > 1) {
                     // create playlist
                     const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
-                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
+                    container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
                 } else if (file_objs.length === 1) {
                     container = file_objs[0];
                 } else {
@@ -370,15 +377,23 @@ async function downloadQueuedFile(download_uid) {
 // helper functions
 
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
+    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
 
     const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
     const globalArgs = config_api.getConfigItem('ytdl_custom_args');
     const useCookies = config_api.getConfigItem('ytdl_use_cookies');
     const is_audio = type === 'audio';
 
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
 
     if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
 
@@ -388,6 +403,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
 
     // video-specific args
     const selectedHeight = options.selectedHeight;
+    const maxHeight = options.maxHeight;
+    const heightParam = selectedHeight || maxHeight;
 
     // audio-specific args
     const maxBitrate = options.maxBitrate;
@@ -401,8 +418,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     if (!is_audio && !is_youtube) {
         // tiktok videos fail when using the default format
         qualityPath = null;
-    } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
-        qualityPath = ['-f', 'bestvideo+bestaudio']
     }
 
     if (customArgs) {
@@ -410,8 +425,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     } else {
         if (customQualityConfiguration) {
             qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
-        } else if (selectedHeight && selectedHeight !== '' && !is_audio) {
+        } else if (heightParam && heightParam !== '' && !is_audio) {
-            qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
+            qualityPath = ['-f', `'(mp4)[height${maxHeight ? '<' : ''}=${heightParam}]`];
         } else if (is_audio) {
             qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
         }
@@ -493,9 +508,11 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.push('-r', rate_limit);
         }
         
-        const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
         if (default_downloader === 'yt-dlp') {
-            downloadConfig.push('--no-clean-infojson');
+            downloadConfig = utils.filterArgs(downloadConfig, ['--print-json']);
+
+            // in yt-dlp -j --no-simulate is preferable
+            downloadConfig.push('--no-clean-info-json', '-j', '--no-simulate');
         }
 
     }
@@ -503,11 +520,11 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
     // filter out incompatible args
     downloadConfig = filterArgs(downloadConfig, is_audio);
 
-    if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
+    if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
     return downloadConfig;
 }
 
-async function getVideoInfoByURL(url, args = [], download_uid = null) {
+exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
     return new Promise(resolve => {
         // remove bad args
         const new_args = [...args];
@@ -562,8 +579,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
 function filterArgs(args, isAudio) {
     const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
     const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
-    const args_to_remove = isAudio ? video_only_args : audio_only_args;
+    return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
-    return args.filter(x => !args_to_remove.includes(x));
 }
 
 async function checkDownloadPercent(download_uid) {
@@ -625,6 +641,6 @@ function getArchiveFolder(fileFolderPath, options, user_uid) {
     } else if (user_uid) {
         return path.join(fileFolderPath, 'archives');
     } else {
-        return path.join(archivePath);
+        return path.join('appdata', 'archives');
     }
 }

backend/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-CMD="pm2-runtime pm2.config.js"
+CMD="npm start"
 
 # if the first arg starts with "-" pass it to program
 if [ "${1#-}" != "$1" ]; then

backend/fix-scripts/001-fix_download_permissions.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # INTERACTIVE PERMISSIONS FIX SCRIPT FOR YTDL-M
 # Date: 2022-05-03
@@ -6,8 +6,7 @@
 # If you want to run this script on a bare-metal installation instead of within Docker
 # make sure that the paths configured below match your paths! (it's wise to use the full paths)
 # USAGE: within your container's bash shell:
-# chmod -R +x ./fix-scripts/
+# ./fix-scripts/<name of fix-script>
-# ./fix-scripts/001-fix_download_permissions.sh
 
 # User defines / Docker env defaults
 PATH_SUBS=/app/subscriptions

backend/fix-scripts/002-fix_dupes_per_archive_file.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M
+# Date: 2022-05-09
+
+# If you want to run this script on a bare-metal installation instead of within Docker
+# make sure that the paths configured below match your paths! (it's wise to use the full paths)
+# USAGE: within your container's bash shell:
+# ./fix-scripts/<name of fix-script>
+
+# User defines (NO TRAILING SLASHES) / Docker env defaults
+PATH_SUBSARCHIVE=/app/subscriptions/archives
+PATH_ONEOFFARCHIVE=/app/appdata/archives
+
+# Backup paths (substitute with your personal preference if you like)
+PATH_SUBSARCHIVEBKP=$PATH_SUBSARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+PATH_ONEOFFARCHIVEBKP=$PATH_ONEOFFARCHIVE-BKP-$(date +%Y%m%d%H%M%S)
+
+
+# Define Colors for TUI
+yellow=$(tput setaf 3)
+normal=$(tput sgr0)
+
+tput civis                                                          # hide the cursor
+
+clear -x
+printf "\n"
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "Welcome to the INTERACTIVE ARCHIVE-DUPE-ENTRY FIX SCRIPT FOR YTDL-M."
+printf "\nThis script will cycle through the archive files in the folders mentioned"
+printf "\nbelow and remove within each archive the dupe entries. (compact them)"
+printf "\nDuring some older builds of YTDL-M the archives could receive dupe"
+printf "\nentries and blow up in size, sometimes causing conflicts with download management."
+printf '\n%*s' "${COLUMNS:-$(tput cols)}" '' | tr ' ' -             # horizontal line
+printf "\n"
+
+# check whether dirs exist
+i=0
+[ -d $PATH_SUBSARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found Subscriptions archive directory at ${PATH_SUBSARCHIVE}"
+[ -d $PATH_ONEOFFARCHIVE ] && i=$((i+1)) && printf "\n✔ (${i}/2) Found one-off archive directory at ${PATH_ONEOFFARCHIVE}"
+
+# Ask to proceed or cancel, exit on missing paths
+case $i in
+  0)
+    printf "\n\n  Couldn't find any archive location path! \n\nPlease edit this script to configure!"
+    tput cnorm
+    exit 2;;
+  2)
+    printf "\n\n  Found all archive locations. \n\nProceed? (Y/N)";;
+  *)
+    printf "\n\n  Only found ${i} out of 2 archive locations! Something about this script's config must be wrong. \n\nProceed anyways? (Y/N)";;
+esac
+old_stty_cfg=$(stty -g)
+stty raw -echo ; answer=$(head -c 1) ; stty $old_stty_cfg          # Careful playing with stty
+if echo "$answer" | grep -iq "^y" ;then
+    printf "\n\nRunning jobs now... (this may take a while)\n"
+
+    printf "\nBacking up directories...\n"
+
+    chars="⣾⣽⣻⢿⡿⣟⣯⣷"
+    cp -R $PATH_SUBSARCHIVE $PATH_SUBSARCHIVEBKP &
+    PID=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.15
+    done
+    [ -d $PATH_SUBSARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_SUBSARCHIVE} to ${PATH_SUBSARCHIVEBKP} ($(du -sh $PATH_SUBSARCHIVEBKP | cut -f1))\n"
+
+    cp -R $PATH_ONEOFFARCHIVE $PATH_ONEOFFARCHIVEBKP &
+    PID2=$!
+    i=1
+    echo -n ' '
+    while [ -d /proc/$PID2 ]
+    do
+      printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+      sleep 0.1
+    done
+    [ -d $PATH_ONEOFFARCHIVEBKP ] && printf "\r✔ Backed up ${PATH_ONEOFFARCHIVE} to ${PATH_ONEOFFARCHIVEBKP} ($(du -sh $PATH_ONEOFFARCHIVEBKP | cut -f1))\n"
+
+
+    printf "\nCompacting files...\n"
+
+    tmpfile=$(mktemp) &&
+
+    [ -d $PATH_SUBSARCHIVE ] &&
+    find $PATH_SUBSARCHIVE -name '*.txt' -print0 | while read -d $'\0' file # Set delimiter to null because we want to catch all possible filenames (WE CANNOT CHANGE IFS HERE) - https://stackoverflow.com/a/15931055
+    do
+        cp "$file" "$tmpfile"
+        { awk '!x[$0]++' "$tmpfile" > "$file"; } &                         # https://unix.stackexchange.com/questions/159695/how-does-awk-a0-work
+        PID3=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID3 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [[ "$AFTER" -ne "$BEFORE" ]]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action needed for file: ${file}\n"
+        fi
+    done
+
+    [ -d $PATH_ONEOFFARCHIVE ] &&
+    find $PATH_ONEOFFARCHIVE -name '*.txt' -print0 | while read -d $'\0' file
+    do
+        cp "$file" "$tmpfile" &
+        awk '!x[$0]++' "$tmpfile" > "$file" &
+        PID4=$!
+        i=1
+        echo -n ''
+        while [ -d /proc/$PID4 ]
+        do
+          printf "${yellow}\b${chars:i++%${#chars}:1}${normal}"
+          sleep 0.1
+        done
+        BEFORE=$(wc -l < $tmpfile)
+        AFTER=$(wc -l < $file)
+        if [ "$BEFORE" -ne "$AFTER" ]; then
+          printf "\b✔ Compacted down to ${AFTER} lines from ${BEFORE}: ${file}\n"
+          else
+          printf "\bℹ No action ran for file: ${file}\n"
+        fi
+    done
+    tput cnorm                                                     # show the cursor
+    rm "$tmpfile"
+
+    printf "\n\n✔ Done."
+    printf "\nℹ Please keep in mind that you may still want to"
+    printf "\n  run corruption checks against your archives!\n\n"
+    exit
+else
+    tput cnorm
+    printf "\nOkay, bye.\n\n"
+    exit
+fi

backend/package.json

@@ -5,20 +5,9 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "nodemon app.js",
+    "start": "pm2-runtime --raw pm2.config.js",
     "debug": "set YTDL_MODE=debug && node app.js"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "*.js",
-      "appdata/*",
-      "public/*"
-    ],
-    "watch": [
-      "restart_update.json",
-      "restart_general.json"
-    ]
-  },
   "repository": {
     "type": "git",
     "url": ""
@@ -47,11 +36,10 @@
     "mocha": "^9.2.2",
     "moment": "^2.29.2",
     "mongodb": "^3.6.9",
-    "multer": "^1.4.2",
+    "multer": "1.4.5-lts.1",
     "node-fetch": "^2.6.7",
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
-    "nodemon": "^2.0.7",
     "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.0",

backend/pm2.config.js

@@ -6,4 +6,4 @@ module.exports = {
       out_file: "/dev/null",
       error_file: "/dev/null"
     }]
-}
+}

backend/subscriptions.js

@@ -178,7 +178,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
+        retrievedID = fs.readJSONSync(jsonPath)['id'];
         await fs.unlink(jsonPath);
     }
 
@@ -196,12 +196,11 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (!deleteForever && useArchive && sub.archive && retrievedID) {
+            if (useArchive && retrievedID) {
-                const archive_path = path.join(sub.archive, 'archive.txt')
+                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
-                // if archive exists, remove line with video ID
+
-                if (await fs.pathExists(archive_path)) {
+                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-                    utils.removeIDFromArchive(archive_path, retrievedID);
+                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
-                }
             }
             return true;
         }
@@ -242,64 +241,22 @@ async function getVideosForSub(sub, user_uid = null) {
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable')) {
+                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        // TODO: reimplement
+                        const outputs = err.stdout.split(/\r\n|\r|\n/);
-
+                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
-                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        resolve(files_to_download);
-                        // for (let i = 0; i < outputs.length; i++) {
-                        //     const output = JSON.parse(outputs[i]);
-                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                        //         // we found a video that errored! add it to the archive to prevent future errors
-                        //         if (sub.archive) {
-                        //             archive_dir = sub.archive;
-                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                        //             fs.appendFileSync(archive_path, output['id']);
-                        //         }
-                        //     }
-                        // }
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
                     }
+                } else {
+                    logger.error('Subscription check failed!');
                 }
                 resolve(false);
             } else if (output) {
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                const files_to_download = await handleOutputJSON(output, sub, user_uid);
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
-                    logger.verbose('No additional videos to download for ' + sub.name);
-                    resolve(true);
-                    return;
-                }
-
-                const output_jsons = [];
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        output_json = JSON.parse(output[i]);
-                        output_jsons.push(output_json);
-                    } catch(e) {
-                        output_json = null;
-                    }
-                    if (!output_json) {
-                        continue;
-                    }
-                }
-
-                const files_to_download = await getFilesToDownload(sub, output_jsons);
-                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
-
-                for (let j = 0; j < files_to_download.length; j++) {
-                    const file_to_download = files_to_download[j];
-                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
-                }
-
                 resolve(files_to_download);
            }
         });
@@ -309,6 +266,43 @@ async function getVideosForSub(sub, user_uid = null) {
     });
 }
 
+async function handleOutputJSON(output, sub, user_uid) {
+    if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+        await setFreshUploads(sub, user_uid);
+        checkVideosForFreshUploads(sub, user_uid);
+    }
+
+    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+        logger.verbose('No additional videos to download for ' + sub.name);
+        return [];
+    }
+
+    const output_jsons = [];
+    for (let i = 0; i < output.length; i++) {
+        let output_json = null;
+        try {
+            output_json = JSON.parse(output[i]);
+            output_jsons.push(output_json);
+        } catch(e) {
+            output_json = null;
+        }
+        if (!output_json) {
+            continue;
+        }
+    }
+
+    const files_to_download = await getFilesToDownload(sub, output_jsons);
+    const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+    for (let j = 0; j < files_to_download.length; j++) {
+        const file_to_download = files_to_download[j];
+        file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']);  // prevent download object from blowing up in size
+        await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
+    }
+
+    return files_to_download;
+}
+
 function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let basePath = null;
     if (user_uid)
@@ -319,10 +313,10 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) {
     let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
 
     const base_download_options = {
-        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        maxHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
         customFileFolderPath: getAppendedBasePath(sub, basePath),
         customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        customArchivePath: path.join(basePath, 'archives', sub.name),
         additionalArgs: sub.custom_args
     }
 
@@ -389,11 +383,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push('--download-archive', archive_path);
     }
 
-    // if streaming only mode, just get the list of videos
-    if (sub.streamingOnly) {
-        downloadConfig = ['-f', 'best', '--dump-json'];
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -418,9 +407,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
     if (default_downloader === 'yt-dlp') {
-        downloadConfig.push('--no-clean-infojson');
+        downloadConfig.push('--no-clean-info-json');
     }
 
+    downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
+
     return downloadConfig;
 }
 
@@ -467,7 +458,7 @@ async function updateSubscription(sub) {
 
 async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
     subs.forEach(async sub => {
-        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
+        await updateSubscriptionProperty(sub, assignment_obj);
     });
 }
 
@@ -479,6 +470,7 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
 
 async function setFreshUploads(sub) {
     const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
+    if (!sub_files) return;
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
     sub_files.forEach(async file => {
         if (current_date === file['upload_date'].replace(/-/g, '')) {

backend/tasks.js

@@ -148,6 +148,7 @@ exports.updateTaskSchedule = async (task_key, schedule) => {
     await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
     if (TASKS[task_key]['job']) {
         TASKS[task_key]['job'].cancel();
+        TASKS[task_key]['job'] = null;
     }
     if (schedule) {
         TASKS[task_key]['job'] = scheduleJob(task_key, schedule);

backend/test/tests.js

@@ -1,6 +1,7 @@
-var assert = require('assert');
+const assert = require('assert');
 const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
 
 process.chdir('./backend')
 
@@ -39,9 +40,29 @@ const utils = require('../utils');
 const subscriptions_api = require('../subscriptions');
 const fs = require('fs-extra');
 const { uuid } = require('uuidv4');
+const NodeID3 = require('node-id3');
 
 db_api.initialize(db, users_db);
 
+const sample_video_json = {
+    id: "Sample Video",
+    title: "Sample Video",
+    thumbnailURL: "https://sampleurl.jpg",
+    isAudio: false,
+    duration: 177.413,
+    url: "sampleurl.com",
+    uploader: "Sample Uploader",
+    size: 2838445,
+    path: "users\\admin\\video\\Sample Video.mp4",
+    upload_date: "2017-07-28",
+    description: null,
+    view_count: 230,
+    abr: 128,
+    thumbnailPath: null,
+    user_uid: "admin",
+    uid: "1ada04ab-2773-4dd4-bbdd-3e2d40761c50",
+    registered: 1628469039377
+}
 
 describe('Database', async function() {
     describe('Import', async function() {
@@ -214,7 +235,7 @@ describe('Database', async function() {
             for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
                 const uid = uuid();
                 if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
-                test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J  - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon:  http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+                test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
             }
             const insert_start = Date.now();
             let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
@@ -235,6 +256,30 @@ describe('Database', async function() {
             assert(success);
         });
     });
+
+    describe('Local DB Filters', async function() {
+        it('Basic', async function() {
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: 'test'}, 'find');
+            assert(result && result['test'] === 'test');
+        });
+
+        it('Regex', async function() {
+            const filter = {$regex: `\\w+\\d`, $options: 'i'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Not equals', async function() {
+            const filter = {$ne: 'test'};
+            const result = db_api.applyFilterLocalDB([{test: 'test'}, {test: 'test1'}], {test: filter}, 'find');
+            assert(result && result['test'] === 'test1');
+        });
+
+        it('Nested', async function() {
+            const result = db_api.applyFilterLocalDB([{test1: {test2: 'test3'}}, {test4: 'test5'}], {'test1.test2': 'test3'}, 'find');
+            assert(result && result['test1']['test2'] === 'test3');
+        });
+    })
 });
 
 describe('Multi User', async function() {
@@ -253,10 +298,12 @@ describe('Multi User', async function() {
             assert(user);
         });
     });
-    describe('Video player - normal', function() {
+    describe('Video player - normal', async function() {
-        const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
+        await db_api.removeRecord('files', {uid: sample_video_json['uid']});
+        await db_api.insertRecordIntoTable('files', sample_video_json);
+        const video_to_test = sample_video_json['uid'];
         it('Get video', async function() {
-            const video_obj = db_api.getVideo(video_to_test, 'admin');
+            const video_obj = await db_api.getVideo(video_to_test);
             assert(video_obj);
         });
 
@@ -341,7 +388,9 @@ describe('Downloader', function() {
     });
 
     it('Get file info', async function() {
-
+        this.timeout(300000);
+        const info = await downloader_api.getVideoInfoByURL(url);
+        assert(!!info);
     });
 
     it('Download file', async function() {
@@ -352,6 +401,19 @@ describe('Downloader', function() {
 
     });
 
+    it('Tag file', async function() {
+        const audio_path = './test/sample.mp3';
+        const sample_json = fs.readJSONSync('./test/sample.info.json');
+        const tags = {
+            title: sample_json['title'],
+            artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
+            TRCK: '27'
+        }
+        NodeID3.write(tags, audio_path);
+        const written_tags = NodeID3.read(audio_path);
+        assert(written_tags['raw']['TRCK'] === '27');
+    });
+
     it('Queue file', async function() {
         this.timeout(300000); 
         const returned_download = await downloader_api.createDownload(url, 'video', options);
@@ -360,20 +422,23 @@ describe('Downloader', function() {
     });
 
     it('Pause file', async function() {
-
+        const returned_download = await downloader_api.createDownload(url, 'video', options);
+        await downloader_api.pauseDownload(returned_download['uid']);
+        const updated_download = await db_api.getRecord('download_queue', {uid: returned_download['uid']});
+        assert(updated_download['paused'] && !updated_download['running']);
     });
 
     it('Generate args', async function() {
         const args = await downloader_api.generateArgs(url, 'video', options);
-        console.log(args);
+        assert(args.length > 0);
     });
 
     it('Generate args - subscription', async function() {
-        subscriptions_api.initialize(db_api, logger);
         const sub = await subscriptions_api.getSubscription(sub_id);
         const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
-        const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        const args_normal = await downloader_api.generateArgs(url, 'video', options);
-        console.log(args);
+        const args_sub = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
+        console.log(JSON.stringify(args_normal) !== JSON.stringify(args_sub));
     });
 
     it('Generate kodi NFO file', async function() {
@@ -401,6 +466,20 @@ describe('Downloader', function() {
         console.log(updated_args2);
         assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
     });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1493770675';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
+    });
 });
 
 describe('Tasks', function() {
@@ -417,7 +496,7 @@ describe('Tasks', function() {
         };
         tasks_api.TASKS['dummy_task'] = dummy_task;
 
-        await tasks_api.initialize();
+        await tasks_api.setupTasks();
     });
     it('Backup db', async function() {
         const backups_original = await utils.recFindByExt('appdata', 'bak');
@@ -429,12 +508,13 @@ describe('Tasks', function() {
     });
 
     it('Check for missing files', async function() {
+        this.timeout(300000);
         await db_api.removeAllRecords('files', {uid: 'test'});
         const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
         await db_api.insertRecordIntoTable('files', test_missing_file);
         await tasks_api.executeTask('missing_files_check');
-        const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
+        const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
-        assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
+        assert(!missing_file_db_record, true);
     });
 
     it('Check for duplicate files', async function() {
@@ -447,10 +527,13 @@ describe('Tasks', function() {
         await db_api.insertRecordIntoTable('files', test_duplicate_file1);
         await db_api.insertRecordIntoTable('files', test_duplicate_file2);
         await db_api.insertRecordIntoTable('files', test_duplicate_file3);
-        await tasks_api.executeTask('duplicate_files_check');
+
+        await tasks_api.executeRun('duplicate_files_check');
         const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
-        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
         assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
+
+        await tasks_api.executeTask('duplicate_files_check');
+        const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
         assert(duplicated_record_count == 1, true);
     });
 
@@ -475,22 +558,72 @@ describe('Tasks', function() {
     });
 
     it('Schedule and cancel task', async function() {
-        const today_4_hours = new Date();
+        this.timeout(5000);
-        today_4_hours.setHours(today_4_hours.getHours() + 4);
+        const today_one_year = new Date();
-        await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
+        today_one_year.setFullYear(today_one_year.getFullYear() + 1);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+        const schedule_obj = {
+            type: 'timestamp',
+            data: { timestamp: today_one_year.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        const dummy_task = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
+        assert(!!dummy_task['schedule']);
+
         await tasks_api.updateTaskSchedule('dummy_task', null);
-        assert(!!tasks_api.TASKS['dummy_task']['job'], false);
+        const dummy_task_updated = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(!tasks_api.TASKS['dummy_task']['job']);
+        assert(!dummy_task_updated['schedule']);
     });
 
     it('Schedule and run task', async function() {
         this.timeout(5000);
         const today_1_second = new Date();
         today_1_second.setSeconds(today_1_second.getSeconds() + 1);
-        await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
+        const schedule_obj = {
-        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+            type: 'timestamp',
+            data: { timestamp: today_1_second.getTime() }
+        }
+        await tasks_api.updateTaskSchedule('dummy_task', schedule_obj);
+        assert(!!tasks_api.TASKS['dummy_task']['job']);
         await utils.wait(2000);
         const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
-        assert(dummy_task_obj['data'], true);
+        assert(dummy_task_obj['data']);
+    });
+});
+
+describe('Archive', async function() {
+    const archive_path = path.join('test', 'archives');
+    fs.ensureDirSync(archive_path);
+    const archive_file_path = path.join(archive_path, 'archive_video.txt');
+    const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
+    beforeEach(async function() {
+        if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
+        fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
+
+        if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
+        fs.writeFileSync(blacklist_file_path, '');
+    });
+    
+    it('Delete from archive', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
+        const new_archive = fs.readFileSync(archive_file_path);
+        assert(!new_archive.includes('testing2'));
+    });
+
+    it('Delete from archive - blacklist', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
+        const new_archive = fs.readFileSync(archive_file_path);
+        const new_blacklist = fs.readFileSync(blacklist_file_path);
+        assert(!new_archive.includes('testing2'));
+        assert(new_blacklist.includes('testing2'));
+    });
+});
+
+describe('Utils', async function() {
+    it('Strip properties', async function() {
+        const test_obj = {test1: 'test1', test2: 'test2', test3: 'test3'};
+        const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
+        assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
     });
 });

backend/twitch.js

@@ -1,90 +1,64 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
 const config_api = require('./config');
+const logger = require('./logger');
 
-async function getCommentsForVOD(clientID, vodId) {
+const moment = require('moment');
-    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
+const fs = require('fs-extra')
-        batch,
+const path = require('path');
-        cursor;
 
-    let comments = null;
+async function getCommentsForVOD(clientID, clientSecret, vodId) {
-
+    const { promisify } = require('util');
-    try {
+    const child_process = require('child_process');
-        do {
+    const exec = promisify(child_process.exec);
-            batch = (await Axios.get(url, {
+    
-                headers: {
+    // Reject invalid params to prevent command injection attack
-                    'Client-ID': clientID,
+    if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
-                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
+        logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
-                    'Content-Type': 'application/json; charset=UTF-8',
+        return null;
-                }
-            })).data;
-
-            const str = batch.comments.map(c => {
-                let {
-                        created_at: msgCreated,
-                        content_offset_seconds: timestamp,
-                        commenter: {
-                            name,
-                            _id,
-                            created_at: acctCreated
-                        },
-                        message: {
-                            body: msg,
-                            user_color: user_color
-                        }
-                    } = c;
-
-                const timestamp_str = moment.duration(timestamp, 'seconds')
-                    .toISOString()
-                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
-                        (_, ...ms) => {
-                            const seg = v => v ? v.padStart(2, '0') : '00';
-                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
-                        });
-
-                acctCreated = moment(acctCreated).utc();
-                msgCreated = moment(msgCreated).utc();
-
-                if (!comments) comments = [];
-
-                comments.push({
-                    timestamp: timestamp,
-                    timestamp_str: timestamp_str,
-                    name: name,
-                    message: msg,
-                    user_color: user_color
-                });
-                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
-                // return line;
-            }).join('\n');
-
-            cursor = batch._next;
-            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
-            await new Promise(res => setTimeout(res, 300));
-        } while (cursor);
-    } catch (err) {
-        console.error(err);
     }
 
-    return comments;
+    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
+
+    if (result['stderr']) {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
+        logger.error(result['stderr']);
+        return null;
+    }
+
+    const temp_chat_path = path.join('appdata', `${vodId}.json`);
+
+    const raw_json = fs.readJSONSync(temp_chat_path);
+    const new_json = raw_json.comments.map(comment_obj => {
+        return {
+            timestamp: comment_obj.content_offset_seconds,
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+            name: comment_obj.commenter.name,
+            message: comment_obj.message.body,
+            user_color: comment_obj.message.user_color
+        }
+    });
+
+    fs.unlinkSync(temp_chat_path);
+
+    return new_json;
 }
 
 async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
     let file_path = null;
 
     if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
         }
     }
 
@@ -96,23 +70,28 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
     return chat_file;
 }
 
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
-    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
-    const chat = await getCommentsForVOD(twitch_api_key, vodId);
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
 
     // save file if needed params are included
     let file_path = null;
-    if (user_uid) {
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            file_path = path.join(type, `${id}.twitch_chat.json`);
         }
     }
 
@@ -121,6 +100,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
     return chat;
 }
 
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
 module.exports = {
     getCommentsForVOD: getCommentsForVOD,
     getTwitchChatByFileID: getTwitchChatByFileID,

backend/utils.js

@@ -172,11 +172,13 @@ function getExpectedFileSize(input_info_jsons) {
         const formats = info_json['format_id'].split('+');
         let individual_expected_filesize = 0;
         formats.forEach(format_id => {
-            info_json.formats.forEach(available_format => {
+            if (info_json.formats !== undefined) {
-                if (available_format.format_id === format_id && available_format.filesize) {
+                info_json.formats.forEach(available_format => {
-                    individual_expected_filesize += available_format.filesize;
+                  if (available_format.format_id === format_id && (available_format.filesize || available_format.filesize_approx)) {
-                }
+                    individual_expected_filesize += (available_format.filesize ? available_format.filesize : available_format.filesize_approx);
-            });
+                  }
+                });
+            }
         });
         expected_filesize += individual_expected_filesize;
     });
@@ -210,7 +212,7 @@ function deleteJSONFile(file_path, type) {
     const ext = type === 'audio' ? '.mp3' : '.mp4';
 
     const file_path_no_extension = removeFileExtension(file_path);
-    
+
     let json_path = file_path_no_extension + '.info.json';
     let alternate_json_path = file_path_no_extension + ext + '.info.json';
 
@@ -218,8 +220,11 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-async function removeIDFromArchive(archive_path, id) {
+// archive helper functions
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
+
+async function removeIDFromArchive(archive_path, type, id) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
     if (!data) {
         logger.error('Archive could not be found.');
         return;
@@ -236,12 +241,34 @@ async function removeIDFromArchive(archive_path, id) {
         }
     }
 
+    if (lastIndex === -1) return null;
+
     const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
 
     // UPDATE FILE WITH NEW DATA
     const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
+    await fs.writeFile(archive_file, updatedData);
-    if (line) return line;
+    if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
+}
+
+async function writeToBlacklist(archive_folder, type, line) {
+    let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
+    // adds newline to the beginning of the line
+    line.replace('\n', '');
+    line.replace('\r', '');
+    line = '\n' + line;
+    await fs.appendFile(blacklistPath, line);
+}
+
+async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    if (await fs.pathExists(archive_path)) {
+        const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
+        if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
+    } else {
+        logger.info(`Could not find archive file for file ${uid}. Creating...`);
+        await fs.close(await fs.open(archive_file, 'w'));
+    }
 }
 
 function durationStringToNumber(dur_str) {
@@ -306,9 +333,9 @@ function createEdgeNGrams(str) {
     if (str && str.length > 3) {
         const minGram = 3
         const maxGram = str.length
-        
+
         return str.split(" ").reduce((ngrams, token) => {
-            if (token.length > minGram) {   
+            if (token.length > minGram) {
                 for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
                     ngrams = [...ngrams, token.substr(0, i)]
                 }
@@ -318,7 +345,7 @@ function createEdgeNGrams(str) {
             return ngrams
         }, []).join(" ")
     }
-    
+
     return str
 }
 
@@ -418,7 +445,7 @@ async function fetchFile(url, path, file_label) {
 async function restartServer(is_update = false) {
     logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
 
-    // the following line restarts the server through nodemon
+    // the following line restarts the server through pm2
     fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
     process.exit(1);
 }
@@ -434,7 +461,7 @@ function injectArgs(original_args, new_args) {
         for (let i = 0; i < new_args.length; i++) {
             const new_arg = new_args[i];
             if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
-            
+
             if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
                 if (original_args.includes(new_arg)) {
                     const original_index = original_args.indexOf(new_arg);
@@ -456,6 +483,60 @@ function injectArgs(original_args, new_args) {
     return updated_args;
 }
 
+function filterArgs(args, args_to_remove) {
+    return args.filter(x => !args_to_remove.includes(x));
+}
+
+const searchObjectByString = function(o, s) {
+    s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
+    s = s.replace(/^\./, '');           // strip a leading dot
+    var a = s.split('.');
+    for (var i = 0, n = a.length; i < n; ++i) {
+        var k = a[i];
+        if (k in o) {
+            o = o[k];
+        } else {
+            return;
+        }
+    }
+    return o;
+}
+
+function stripPropertiesFromObject(obj, properties, whitelist = false) {
+    if (!whitelist) {
+        const new_obj = JSON.parse(JSON.stringify(obj));
+        for (let field of properties) {
+            delete new_obj[field];
+        }
+        return new_obj;
+    }
+
+    const new_obj = {};
+    for (let field of properties) {
+        new_obj[field] = obj[field];
+    }
+    return new_obj;
+}
+
+function getArchiveFolder(type, user_uid = null, sub = null) {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -485,11 +566,12 @@ module.exports = {
     fixVideoMetadataPerms: fixVideoMetadataPerms,
     deleteJSONFile: deleteJSONFile,
     removeIDFromArchive: removeIDFromArchive,
+    writeToBlacklist: writeToBlacklist,
+    deleteFileFromArchive: deleteFileFromArchive,
     getDownloadedFilesByType: getDownloadedFilesByType,
     createContainerZipFile: createContainerZipFile,
     durationStringToNumber: durationStringToNumber,
     getMatchingCategoryFiles: getMatchingCategoryFiles,
-    addUIDsToCategory: addUIDsToCategory,
     getCurrentDownloader: getCurrentDownloader,
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
@@ -501,5 +583,9 @@ module.exports = {
     fetchFile: fetchFile,
     restartServer: restartServer,
     injectArgs: injectArgs,
+    filterArgs: filterArgs,
+    searchObjectByString: searchObjectByString,
+    stripPropertiesFromObject: stripPropertiesFromObject,
+    getArchiveFolder: getArchiveFolder,
     File: File
 }

backend/youtube-dl.js

@@ -90,7 +90,7 @@ exports.updateYoutubeDL = async (latest_update_version) => {
 
 exports.verifyBinaryExistsLinux = () => {
     const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
-    if (!is_windows && details_json && details_json['path'].includes('.exe')) {
+    if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
         details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
         details_json['exec'] = 'youtube-dl';
         details_json['version'] = OUTDATED_VERSION;

chart/Chart.yaml

@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "4.2"
+appVersion: "4.3"

docker-compose.yml

@@ -2,7 +2,6 @@ version: "2"
 services:
     ytdl_material:
         environment: 
-            ALLOW_CONFIG_MUTATIONS: 'true'
             ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
             ytdl_use_local_db: 'false'
             write_ytdl_config: 'true'
@@ -17,14 +16,12 @@ services:
             - ./users:/app/users
         ports:
             - "8998:17442"
-        image: tzahi12345/youtubedl-material:nightly
+        image: tzahi12345/youtubedl-material:latest
     ytdl-mongo-db:
         image: mongo
-        ports:
-            - "27017:27017"
         logging:
             driver: "none"          
         container_name: mongo-db
         restart: always
         volumes:
-            - ./db/:/data/db
+            - ./db/:/data/db

ffmpeg-fetch.sh

@@ -40,4 +40,4 @@ echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
 cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
 cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
 echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
-rm -rf /tmp/ffmpeg ffmpeg.txz
+rm -rf /tmp/ffmpeg ffmpeg.txz

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-dl-material",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "license": "MIT",
   "scripts": {
     "ng": "ng",
@@ -13,7 +13,7 @@
     "e2e": "ng e2e",
     "electron": "ng build --base-href ./ && electron .",
     "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
-    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n"
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n --out-file=messages.en.xlf"
   },
   "engines": {
     "node": "12.3.1",
@@ -21,18 +21,18 @@
   },
   "private": true,
   "dependencies": {
-    "@angular-devkit/core": "^13.3.3",
+    "@angular-devkit/core": "^14.0.4",
-    "@angular/animations": "^13.3.4",
+    "@angular/animations": "^14.0.4",
-    "@angular/cdk": "^13.3.4",
+    "@angular/cdk": "^14.0.4",
-    "@angular/common": "^13.3.4",
+    "@angular/common": "^14.0.4",
-    "@angular/compiler": "^13.3.4",
+    "@angular/compiler": "^14.0.4",
-    "@angular/core": "^13.3.4",
+    "@angular/core": "^14.0.4",
-    "@angular/forms": "^13.3.4",
+    "@angular/forms": "^14.0.4",
-    "@angular/localize": "^13.3.4",
+    "@angular/localize": "^14.0.4",
-    "@angular/material": "^13.3.4",
+    "@angular/material": "^14.0.4",
-    "@angular/platform-browser": "^13.3.4",
+    "@angular/platform-browser": "^14.0.4",
-    "@angular/platform-browser-dynamic": "^13.3.4",
+    "@angular/platform-browser-dynamic": "^14.0.4",
-    "@angular/router": "^13.3.4",
+    "@angular/router": "^14.0.4",
     "@fontsource/material-icons": "^4.5.4",
     "@ngneat/content-loader": "^5.0.0",
     "@videogular/ngx-videogular": "^5.0.1",
@@ -55,10 +55,10 @@
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^13.3.3",
+    "@angular-devkit/build-angular": "^14.0.4",
-    "@angular/cli": "^13.3.3",
+    "@angular/cli": "^14.0.4",
-    "@angular/compiler-cli": "^13.3.4",
+    "@angular/compiler-cli": "^14.0.4",
-    "@angular/language-service": "^13.3.4",
+    "@angular/language-service": "^14.0.4",
     "@types/core-js": "^2.5.2",
     "@types/file-saver": "^2.0.1",
     "@types/jasmine": "~3.6.0",
@@ -66,7 +66,7 @@
     "@typescript-eslint/eslint-plugin": "^4.29.0",
     "@typescript-eslint/parser": "^4.29.0",
     "codelyzer": "^6.0.0",
-    "electron": "^13.6.6",
+    "electron": "^19.0.6",
     "eslint": "^7.32.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",

src/api-types/index.ts

@@ -4,6 +4,7 @@
 
 export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
 export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
+export type { binary } from './models/binary';
 export type { body_19 } from './models/body_19';
 export type { body_20 } from './models/body_20';
 export type { Category } from './models/Category';
@@ -12,6 +13,7 @@ export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermission
 export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
 export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
 export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
+export type { ClearDownloadsRequest } from './models/ClearDownloadsRequest';
 export type { ConcurrentStream } from './models/ConcurrentStream';
 export type { Config } from './models/Config';
 export type { ConfigResponse } from './models/ConfigResponse';
@@ -23,6 +25,7 @@ export type { CropFileSettings } from './models/CropFileSettings';
 export type { DatabaseFile } from './models/DatabaseFile';
 export { DBBackup } from './models/DBBackup';
 export type { DBInfoResponse } from './models/DBInfoResponse';
+export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
 export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
 export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
 export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
@@ -36,13 +39,14 @@ export type { DownloadResponse } from './models/DownloadResponse';
 export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
 export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
 export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
-export type { File } from './models/File';
 export { FileType } from './models/FileType';
+export { FileTypeFilter } from './models/FileTypeFilter';
 export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
 export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
 export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
 export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
 export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
+export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
 export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
 export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
 export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
@@ -80,6 +84,7 @@ export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
 export { Schedule } from './models/Schedule';
 export type { SetConfigRequest } from './models/SetConfigRequest';
 export type { SharingToggle } from './models/SharingToggle';
+export type { Sort } from './models/Sort';
 export type { SubscribeRequest } from './models/SubscribeRequest';
 export type { SubscribeResponse } from './models/SubscribeResponse';
 export type { Subscription } from './models/Subscription';
@@ -98,6 +103,7 @@ export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
 export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
 export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
 export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
+export type { UpdateFileRequest } from './models/UpdateFileRequest';
 export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
 export type { UpdaterStatus } from './models/UpdaterStatus';
 export type { UpdateServerRequest } from './models/UpdateServerRequest';

src/api-types/models/AddFileToPlaylistRequest.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type AddFileToPlaylistRequest = {
-export interface AddFileToPlaylistRequest {
     file_uid: string;
     playlist_id: string;
-}
+};

src/api-types/models/BaseChangePermissionsRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { UserPermission } from './UserPermission';
+import type { UserPermission } from './UserPermission';
-import { YesNo } from './YesNo';
+import type { YesNo } from './YesNo';
 
-export interface BaseChangePermissionsRequest {
+export type BaseChangePermissionsRequest = {
     permission: UserPermission;
     new_value: YesNo;
-}
+};

src/api-types/models/Category.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { CategoryRule } from './CategoryRule';
+import type { CategoryRule } from './CategoryRule';
 
-export interface Category {
+export type Category = {
     name?: string;
     uid?: string;
     rules?: Array<CategoryRule>;
@@ -12,4 +12,4 @@ export interface Category {
      * Overrides file output for downloaded files in category
      */
     custom_output?: string;
-}
+};

src/api-types/models/CategoryRule.ts

@@ -2,11 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CategoryRule = {
-export interface CategoryRule {
     preceding_operator?: CategoryRule.preceding_operator;
     comparator?: CategoryRule.comparator;
-}
+};
 
 export namespace CategoryRule {
 

src/api-types/models/ChangeRolePermissionsRequest.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
 
-export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
+export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
-    role: string;
+role: string;
-}
+});

src/api-types/models/ChangeUserPermissionsRequest.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
+import type { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
 
-export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
+export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
-    user_uid: string;
+user_uid: string;
-}
+});

src/api-types/models/CheckConcurrentStreamRequest.ts

@@ -2,10 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CheckConcurrentStreamRequest = {
-export interface CheckConcurrentStreamRequest {
     /**
      * UID of the concurrent stream
      */
     uid: string;
-}
+};

src/api-types/models/CheckConcurrentStreamResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { ConcurrentStream } from './ConcurrentStream';
+import type { ConcurrentStream } from './ConcurrentStream';
 
-export interface CheckConcurrentStreamResponse {
+export type CheckConcurrentStreamResponse = {
     stream: ConcurrentStream;
-}
+};

src/api-types/models/ClearDownloadsRequest.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type ClearDownloadsRequest = {
+    clear_finished?: boolean;
+    clear_paused?: boolean;
+    clear_errors?: boolean;
+};

src/api-types/models/ConcurrentStream.ts

@@ -2,9 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type ConcurrentStream = {
-export interface ConcurrentStream {
     playback_timestamp?: number;
     unix_timestamp?: number;
     playing?: boolean;
-}
+};

src/api-types/models/Config.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type Config = {
-export interface Config {
     YoutubeDLMaterial: any;
-}
+};

src/api-types/models/ConfigResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Config } from './Config';
+import type { Config } from './Config';
 
-export interface ConfigResponse {
+export type ConfigResponse = {
     config_file: Config;
     success: boolean;
-}
+};

src/api-types/models/CreateCategoryRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CreateCategoryRequest = {
-export interface CreateCategoryRequest {
     name: string;
-}
+};

src/api-types/models/CreateCategoryResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Category } from './Category';
+import type { Category } from './Category';
 
-export interface CreateCategoryResponse {
+export type CreateCategoryResponse = {
     new_category?: Category;
     success?: boolean;
-}
+};

src/api-types/models/CreatePlaylistRequest.ts

@@ -2,11 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+export type CreatePlaylistRequest = {
-
-export interface CreatePlaylistRequest {
     playlistName: string;
     uids: Array<string>;
-    type: FileType;
     thumbnailURL: string;
-}
+};

src/api-types/models/CreatePlaylistResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface CreatePlaylistResponse {
+export type CreatePlaylistResponse = {
     new_playlist: Playlist;
     success: boolean;
-}
+};

src/api-types/models/CropFileSettings.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type CropFileSettings = {
-export interface CropFileSettings {
     cropFileStart: number;
     cropFileEnd: number;
-}
+};

src/api-types/models/DBBackup.ts

@@ -2,13 +2,12 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DBBackup = {
-export interface DBBackup {
     name: string;
     timestamp: number;
     size: number;
     source: DBBackup.source;
-}
+};
 
 export namespace DBBackup {
 

src/api-types/models/DBInfoResponse.ts

@@ -2,17 +2,17 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { TableInfo } from './TableInfo';
+import type { TableInfo } from './TableInfo';
 
-export interface DBInfoResponse {
+export type DBInfoResponse = {
     using_local_db?: boolean;
     stats_by_table?: {
-files?: TableInfo,
+files?: TableInfo;
-playlists?: TableInfo,
+playlists?: TableInfo;
-categories?: TableInfo,
+categories?: TableInfo;
-subscriptions?: TableInfo,
+subscriptions?: TableInfo;
-users?: TableInfo,
+users?: TableInfo;
-roles?: TableInfo,
+roles?: TableInfo;
-download_queue?: TableInfo,
+download_queue?: TableInfo;
 };
-}
+};

src/api-types/models/DatabaseFile.ts

@@ -2,11 +2,16 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { Category } from './Category';
 
-export interface DatabaseFile {
+export type DatabaseFile = {
     id: string;
     title: string;
+    /**
+     * Backup if thumbnailPath is not defined
+     */
     thumbnailURL: string;
+    thumbnailPath?: string;
     isAudio: boolean;
     /**
      * In seconds
@@ -14,9 +19,25 @@ export interface DatabaseFile {
     duration: number;
     url: string;
     uploader: string;
+    /**
+     * In bytes
+     */
     size: number;
     path: string;
     upload_date: string;
     uid: string;
     sharingEnabled?: boolean;
-}
+    category?: Category;
+    view_count?: number;
+    local_view_count?: number;
+    sub_id?: string;
+    registered?: number;
+    /**
+     * In pixels, only for videos
+     */
+    height?: number;
+    /**
+     * In Kbps
+     */
+    abr?: number;
+};

src/api-types/models/DeleteAllFilesResponse.ts

@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type DeleteAllFilesResponse = {
+    /**
+     * Number of files found matching search parameters
+     */
+    file_count?: number;
+    /**
+     * Number of files removed
+     */
+    delete_count?: number;
+};

src/api-types/models/DeleteCategoryRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteCategoryRequest = {
-export interface DeleteCategoryRequest {
     category_uid: string;
-}
+};

src/api-types/models/DeleteMp3Mp4Request.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteMp3Mp4Request = {
-export interface DeleteMp3Mp4Request {
     uid: string;
     blacklistMode?: boolean;
-}
+};

src/api-types/models/DeletePlaylistRequest.ts

@@ -2,9 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+export type DeletePlaylistRequest = {
-
-export interface DeletePlaylistRequest {
     playlist_id: string;
-    type: FileType;
+};
-}

src/api-types/models/DeleteSubscriptionFileRequest.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { SubscriptionRequestData } from './SubscriptionRequestData';
+import type { SubscriptionRequestData } from './SubscriptionRequestData';
 
-export interface DeleteSubscriptionFileRequest {
+export type DeleteSubscriptionFileRequest = {
     file: string;
     file_uid?: string;
     sub: SubscriptionRequestData;
@@ -12,4 +12,4 @@ export interface DeleteSubscriptionFileRequest {
      * If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
      */
     deleteForever?: boolean;
-}
+};

src/api-types/models/DeleteUserRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DeleteUserRequest = {
-export interface DeleteUserRequest {
     uid: string;
-}
+};

src/api-types/models/Dictionary.ts

@@ -1,7 +0,0 @@
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-
-export type Dictionary<T> = {
-    [key: string]: T;
-}

src/api-types/models/Download.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type Download = {
-export interface Download {
     uid: string;
     ui_uid?: string;
     running: boolean;
@@ -23,4 +22,5 @@ export interface Download {
     user_uid?: string;
     sub_id?: string;
     sub_name?: string;
-}
+    prefetched_info?: any;
+};

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,9 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DownloadArchiveRequest = {
-export interface DownloadArchiveRequest {
     sub: {
-archive_dir: string,
+archive_dir: string;
 };
-}
+};

src/api-types/models/DownloadFileRequest.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface DownloadFileRequest {
+export type DownloadFileRequest = {
     uid?: string;
     uuid?: string;
     sub_id?: string;
     playlist_id?: string;
     url?: string;
     type?: FileType;
-}
+};

src/api-types/models/DownloadRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { CropFileSettings } from './CropFileSettings';
+import type { CropFileSettings } from './CropFileSettings';
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface DownloadRequest {
+export type DownloadRequest = {
     url: string;
     /**
      * Video format code. Overrides other quality options.
@@ -41,4 +41,4 @@ export interface DownloadRequest {
     maxBitrate?: string;
     type?: FileType;
     cropFileSettings?: CropFileSettings;
-}
+};

src/api-types/models/DownloadResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Download } from './Download';
+import type { Download } from './Download';
 
-export interface DownloadResponse {
+export type DownloadResponse = {
     download?: Download;
-}
+};

src/api-types/models/DownloadTwitchChatByVODIDRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
-import { Subscription } from './Subscription';
+import type { Subscription } from './Subscription';
 
-export interface DownloadTwitchChatByVODIDRequest {
+export type DownloadTwitchChatByVODIDRequest = {
     /**
      * File ID
      */
@@ -20,4 +20,4 @@ export interface DownloadTwitchChatByVODIDRequest {
      */
     uuid?: string;
     sub?: Subscription;
-}
+};

src/api-types/models/DownloadTwitchChatByVODIDResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { TwitchChatMessage } from './TwitchChatMessage';
+import type { TwitchChatMessage } from './TwitchChatMessage';
 
-export interface DownloadTwitchChatByVODIDResponse {
+export type DownloadTwitchChatByVODIDResponse = {
     chat: Array<TwitchChatMessage>;
-}
+};

src/api-types/models/DownloadVideosForSubscriptionRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type DownloadVideosForSubscriptionRequest = {
-export interface DownloadVideosForSubscriptionRequest {
     subID: string;
-}
+};

src/api-types/models/FileType.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
 export enum FileType {
     AUDIO = 'audio',
     VIDEO = 'video',

src/api-types/models/FileTypeFilter.ts

@@ -0,0 +1,9 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export enum FileTypeFilter {
+    AUDIO_ONLY = 'audio_only',
+    VIDEO_ONLY = 'video_only',
+    BOTH = 'both',
+}

src/api-types/models/GenerateArgsResponse.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GenerateArgsResponse = {
-export interface GenerateArgsResponse {
     args?: Array<string>;
-}
+};

src/api-types/models/GenerateNewApiKeyResponse.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GenerateNewApiKeyResponse = {
-export interface GenerateNewApiKeyResponse {
     new_api_key: string;
-}
+};

src/api-types/models/GetAllCategoriesResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Category } from './Category';
+import type { Category } from './Category';
 
-export interface GetAllCategoriesResponse {
+export type GetAllCategoriesResponse = {
     categories: Array<Category>;
-}
+};

src/api-types/models/GetAllDownloadsRequest.ts

@@ -2,10 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetAllDownloadsRequest = {
-export interface GetAllDownloadsRequest {
     /**
      * Filters downloads with the array
      */
     uids?: Array<string> | null;
-}
+};

src/api-types/models/GetAllDownloadsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Download } from './Download';
+import type { Download } from './Download';
 
-export interface GetAllDownloadsResponse {
+export type GetAllDownloadsResponse = {
     downloads?: Array<Download>;
-}
+};

src/api-types/models/GetAllFilesRequest.ts

@@ -0,0 +1,20 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { FileTypeFilter } from './FileTypeFilter';
+import type { Sort } from './Sort';
+
+export type GetAllFilesRequest = {
+    sort?: Sort;
+    range?: Array<number>;
+    /**
+     * Filter files by title
+     */
+    text_search?: string;
+    file_type_filter?: FileTypeFilter;
+    /**
+     * Include if you want to filter by subscription
+     */
+    sub_id?: string;
+};

src/api-types/models/GetAllFilesResponse.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DatabaseFile } from './DatabaseFile';
+import type { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetAllFilesResponse {
+export type GetAllFilesResponse = {
     files: Array<DatabaseFile>;
     /**
      * All video playlists
      */
     playlists: Array<Playlist>;
-}
+};

src/api-types/models/GetAllSubscriptionsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Subscription } from './Subscription';
+import type { Subscription } from './Subscription';
 
-export interface GetAllSubscriptionsResponse {
+export type GetAllSubscriptionsResponse = {
     subscriptions: Array<Subscription>;
-}
+};

src/api-types/models/GetAllTasksResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Task } from './Task';
+import type { Task } from './Task';
 
-export interface GetAllTasksResponse {
+export type GetAllTasksResponse = {
     tasks?: Array<Task>;
-}
+};

src/api-types/models/GetDBBackupsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DBBackup } from './DBBackup';
+import type { DBBackup } from './DBBackup';
 
-export interface GetDBBackupsResponse {
+export type GetDBBackupsResponse = {
     tasks?: Array<DBBackup>;
-}
+};

src/api-types/models/GetDownloadRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetDownloadRequest = {
-export interface GetDownloadRequest {
     download_uid: string;
-}
+};

src/api-types/models/GetDownloadResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Download } from './Download';
+import type { Download } from './Download';
 
-export interface GetDownloadResponse {
+export type GetDownloadResponse = {
     download?: Download;
-}
+};

src/api-types/models/GetFileFormatsRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetFileFormatsRequest = {
-export interface GetFileFormatsRequest {
     url?: string;
-}
+};

src/api-types/models/GetFileFormatsResponse.ts

@@ -2,11 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { File } from './File';
+export type GetFileFormatsResponse = {
-
-export interface GetFileFormatsResponse {
     success: boolean;
     result: {
-formats?: Array<any>,
+formats?: Array<any>;
 };
-}
+};

src/api-types/models/GetFileRequest.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface GetFileRequest {
+export type GetFileRequest = {
     /**
      * Video UID
      */
@@ -14,4 +14,4 @@ export interface GetFileRequest {
      * User UID
      */
     uuid?: string;
-}
+};

src/api-types/models/GetFileResponse.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DatabaseFile } from './DatabaseFile';
+import type { DatabaseFile } from './DatabaseFile';
 
-export interface GetFileResponse {
+export type GetFileResponse = {
     success: boolean;
     file?: DatabaseFile;
-}
+};

src/api-types/models/GetFullTwitchChatRequest.ts

@@ -2,10 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
-import { Subscription } from './Subscription';
+import type { Subscription } from './Subscription';
 
-export interface GetFullTwitchChatRequest {
+export type GetFullTwitchChatRequest = {
     /**
      * File ID
      */
@@ -16,4 +16,4 @@ export interface GetFullTwitchChatRequest {
      */
     uuid?: string;
     sub?: Subscription;
-}
+};

src/api-types/models/GetFullTwitchChatResponse.ts

@@ -2,8 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetFullTwitchChatResponse = {
-export interface GetFullTwitchChatResponse {
     success: boolean;
     error?: string;
-}
+};

src/api-types/models/GetLogsRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetLogsRequest = {
-export interface GetLogsRequest {
     lines?: number;
-}
+};

src/api-types/models/GetLogsResponse.ts

@@ -2,11 +2,10 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetLogsResponse = {
-export interface GetLogsResponse {
     /**
      * Number of lines to retrieve from the bottom
      */
     logs?: string;
     success?: boolean;
-}
+};

src/api-types/models/GetMp3sResponse.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DatabaseFile } from './DatabaseFile';
+import type { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetMp3sResponse {
+export type GetMp3sResponse = {
     mp3s: Array<DatabaseFile>;
     /**
      * All audio playlists
      */
     playlists: Array<Playlist>;
-}
+};

src/api-types/models/GetMp4sResponse.ts

@@ -2,13 +2,13 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { DatabaseFile } from './DatabaseFile';
+import type { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetMp4sResponse {
+export type GetMp4sResponse = {
     mp4s: Array<DatabaseFile>;
     /**
      * All video playlists
      */
     playlists: Array<Playlist>;
-}
+};

src/api-types/models/GetPlaylistRequest.ts

@@ -2,11 +2,11 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { FileType } from './FileType';
 
-export interface GetPlaylistRequest {
+export type GetPlaylistRequest = {
     playlist_id: string;
     type?: FileType;
     uuid?: string;
     include_file_metadata?: boolean;
-}
+};

src/api-types/models/GetPlaylistResponse.ts

@@ -2,11 +2,14 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { FileType } from './FileType';
+import type { DatabaseFile } from './DatabaseFile';
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetPlaylistResponse {
+export type GetPlaylistResponse = {
     playlist: Playlist;
-    type: FileType;
     success: boolean;
-}
+    /**
+     * File objects for every uid in the playlist's uids property, in the same order
+     */
+    file_objs?: Array<DatabaseFile>;
+};

src/api-types/models/GetPlaylistsRequest.ts

@@ -2,7 +2,6 @@
 /* tslint:disable */
 /* eslint-disable */
 
-
+export type GetPlaylistsRequest = {
-export interface GetPlaylistsRequest {
     include_categories?: boolean;
-}
+};

src/api-types/models/GetPlaylistsResponse.ts

@@ -2,8 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import { Playlist } from './Playlist';
+import type { Playlist } from './Playlist';
 
-export interface GetPlaylistsResponse {
+export type GetPlaylistsResponse = {
     playlists: Array<Playlist>;
-}
+};