diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3366092 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +db +appdata +audio +video +subscriptions +users \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..c5624c4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,20 @@ +{ + "env": { + "browser": true, + "es2021": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [ + "@typescript-eslint" + ], + "rules": { + } +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..94f04ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** +- YoutubeDL-Material version +- Docker tag: (optional) + +Ideally you'd copy the info as presented on the "About" dialogue +in YoutubeDL-Material. +(for that, click on the three dots on the top right and then +check "installation details". On later versions of YoutubeDL- +Material you will find pretty much all the crucial information +here that we need in most cases!) + +**Additional context** +Add any other context about the problem here. For example, a YouTube link. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d233fde --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]" +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..f9586e8 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/.github/workflows" + schedule: + interval: "daily" + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "npm" + directory: "/backend/" + schedule: + interval: "daily" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9eb71be..c509b68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,15 +15,31 @@ jobs: - name: checkout code uses: actions/checkout@v2 - name: setup node - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 + with: + node-version: '14' + cache: 'npm' - name: install dependencies run: | npm install cd backend npm install sudo npm install -g @angular/cli + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' - name: build - run: ng build --prod + run: npm run build - name: prepare artifact upload shell: pwsh run: | @@ -35,7 +51,7 @@ jobs: Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material - New-Item -Path ./build/youtubedl-material -Name users + New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material - name: upload build artifact @@ -69,17 +85,20 @@ jobs: with: name: youtubedl-material path: ${{runner.temp}}/youtubedl-material + - name: extract tag name + id: tag_name + run: echo ::set-output name=tag_name::${GITHUB_REF#refs/tags/} - name: prepare release asset shell: pwsh - run: Compress-Archive -Path ${{runner.temp}}/youtubedl-material -DestinationPath youtubedl-material-${{ github.ref }}.zip - - name: upload build asset + run: Compress-Archive -Path ${{runner.temp}}/youtubedl-material -DestinationPath youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip + - name: upload release asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./youtubedl-material-${{ github.ref }}.zip - asset_name: youtubedl-material-${{ github.ref }}.zip + asset_path: ./youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip + asset_name: youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip asset_content_type: application/zip - name: upload docker-compose asset uses: actions/upload-release-asset@v1 diff --git a/.github/workflows/close-issue-if-noresponse.yml b/.github/workflows/close-issue-if-noresponse.yml new file mode 100644 index 0000000..64d4756 --- /dev/null +++ b/.github/workflows/close-issue-if-noresponse.yml @@ -0,0 +1,38 @@ +name: No Response + +# Both `issue_comment` and `scheduled` event types are required for this Action +# to work properly. +on: + issue_comment: + types: [created] + schedule: + # Schedule for five minutes after the hour, every hour + - cron: '5 * * * *' + +# By specifying the access of one of the scopes, all of those that are not +# specified are set to 'none'. +permissions: + issues: write + +jobs: + noResponse: + runs-on: ubuntu-latest + if: ${{ github.repository == 'Tzahi12345/YoutubeDL-Material' }} + steps: + - uses: lee-dohm/no-response@v0.5.0 + with: + token: ${{ github.token }} + # Comment to post when closing an Issue for lack of response. Set to `false` to disable + closeComment: > + This issue has been automatically closed because there has been no response + to our request for more information from the original author. With only the + information that is currently in the issue, we don't have enough information + to take action. Please reach out if you have or find the answers we need so + that we can investigate further. We will re-open this issue if you provide us + with the requested information with a comment under this issue. + Thank you for your understanding and for trying to help make this application + a better one! + # Number of days of inactivity before an issue is closed for lack of response. + daysUntilClose: 21 + # Label requiring a response. + responseRequiredLabel: "💬 response-needed" diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml new file mode 100644 index 0000000..2fcf807 --- /dev/null +++ b/.github/workflows/docker-pr.yml @@ -0,0 +1,27 @@ +name: docker-pr + +on: + pull_request: + branches: [master] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + - name: Build docker images + run: docker build . -t tzahi12345/youtubedl-material:nightly-pr \ No newline at end of file diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 96ec2c8..6ace410 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -6,22 +6,75 @@ on: tags: description: 'Docker tags' required: true + release: + types: [published] jobs: build-and-push: runs-on: ubuntu-latest + steps: - name: checkout code uses: actions/checkout@v2 + + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + + - name: Set image tag + id: tags + run: | + if [ "${{ github.event.inputs.tags }}" != "" ]; then + echo "::set-output name=tags::${{ github.event.inputs.tags }}" + elif [ ${{ github.event.action }} == "release" ]; then + echo "::set-output name=tags::${{ github.event.release.tag_name }}" + else + echo "Unknown workflow trigger: ${{ github.event.action }}! Cannot determine default tag." + exit 1 + fi + + - name: Generate Docker image metadata + id: docker-meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }} + ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }} + tags: | + type=raw,value=${{ steps.tags.outputs.tags }} + type=raw,value=latest + - name: setup platform emulator uses: docker/setup-qemu-action@v1 + - name: setup multi-arch docker build uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: build & push images uses: docker/build-push-action@v2 with: @@ -29,4 +82,5 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm,linux/arm64/v8 push: true - tags: ${{ github.event.inputs.tags }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c74f29b..cd0373f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,6 +3,19 @@ name: docker on: push: branches: [master] + paths-ignore: + - '.github/**' + - '.vscode/**' + - 'chrome-extension/**' + - 'releases/**' + - '**/**.md' + - '**.crx' + - '**.pem' + - '.dockerignore' + - '.gitignore' + schedule: + - cron: '34 4 * * 2' + workflow_dispatch: jobs: build-and-push: @@ -10,15 +23,58 @@ jobs: steps: - name: checkout code uses: actions/checkout@v2 + + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + - name: setup platform emulator uses: docker/setup-qemu-action@v1 + - name: setup multi-arch docker build uses: docker/setup-buildx-action@v1 + + - name: Generate Docker image metadata + id: docker-meta + uses: docker/metadata-action@v4 + # Defaults: + # DOCKERHUB_USERNAME : tzahi12345 + # DOCKERHUB_REPO : youtubedl-material + # DOCKERHUB_MASTER_TAG: nightly + with: + images: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }} + ghcr.io/${{ github.repository_owner }}/${{ secrets.DOCKERHUB_REPO }} + tags: | + type=raw,${{secrets.DOCKERHUB_MASTER_TAG}}-{{ date 'YYYY-MM-DD' }} + type=raw,${{secrets.DOCKERHUB_MASTER_TAG}} + type=sha,prefix=sha-,format=short + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: build & push images uses: docker/build-push-action@v2 with: @@ -26,4 +82,5 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm,linux/arm64/v8 push: true - tags: tzahi12345/youtubedl-material:nightly + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 9d2ac87..5adce06 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ !.vscode/extensions.json # misc +/.angular/cache /.sass-cache /connect.lock /coverage @@ -65,4 +66,13 @@ backend/appdata/logs/error.log backend/appdata/users.json backend/users/* backend/appdata/cookies.txt -backend/public \ No newline at end of file +backend/public +src/assets/i18n/*.json + +# User Files +db/ +appdata/ +audio/ +video/ +subscriptions/ +users/ \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..5cfd91d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,11 @@ +{ + "recommendations": [ + "angular.ng-template", + "dbaeumer.vscode-eslint", + "waderyan.gitblame", + "42crunch.vscode-openapi", + "redhat.vscode-yaml", + "christian-kohler.npm-intellisense", + "hbenl.vscode-mocha-test-adapter" + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..aa1390b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "mochaExplorer.files": "backend/test/**/*.js", + "mochaExplorer.cwd": "backend", + "mochaExplorer.globImplementation": "vscode", + "mochaExplorer.env": { + "YTDL_MODE": "debug" + } +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..fc1250e --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,60 @@ +{ + "version": "2.0.0", + "windows": { + "options": { + "shell": { + "executable": "cmd.exe", + "args": [ + "/d", "/c" + ] + } + } + }, + "tasks": [ + { + "type": "npm", + "script": "start", + "problemMatcher": [], + "label": "Dev: start frontend", + "detail": "ng serve", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": true, + "clear": false + } + }, + { + "label": "Dev: start backend", + "type": "shell", + "command": "node app.js", + "options": { + "cwd": "./backend", + "env": { + "YTDL_MODE": "debug" + } + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [], + "dependsOn": ["Dev: post-build"] + }, + { + "label": "Dev: post-build", + "type": "shell", + "command": "node src/postbuild.mjs" + }, + { + "label": "Dev: run all", + "dependsOn": ["Dev: start backend", "Dev: start frontend"] + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 75b22d3..0228176 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,81 @@ -FROM alpine:3.12 as frontend +# Fetching our ffmpeg +FROM ubuntu:22.04 AS ffmpeg +ENV DEBIAN_FRONTEND=noninteractive +# Use script due local build compability +COPY docker-utils/ffmpeg-fetch.sh . +RUN chmod +x ffmpeg-fetch.sh +RUN sh ./ffmpeg-fetch.sh -RUN apk add --no-cache \ - npm +# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021) +# Go to 20.04 +FROM ubuntu:20.04 AS base +ARG DEBIAN_FRONTEND=noninteractive +ENV UID=1000 +ENV GID=1000 +ENV USER=youtube +ENV NO_UPDATE_NOTIFIER=true +ENV PM2_HOME=/app/pm2 +ENV ALLOW_CONFIG_MUTATIONS=true +RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \ + apt update && \ + apt install -y --no-install-recommends curl ca-certificates tzdata && \ + curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \ + apt install -y --no-install-recommends nodejs && \ + npm -g install npm n && \ + n 16.14.2 && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* + + +# Build frontend +FROM base as frontend RUN npm install -g @angular/cli - WORKDIR /build -COPY [ "package.json", "package-lock.json", "/build/" ] -RUN npm install - -COPY [ "angular.json", "tsconfig.json", "/build/" ] +COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ] COPY [ "src/", "/build/src/" ] -RUN ng build --prod +RUN npm install && \ + npm run build && \ + ls -al /build/backend/public -#--------------# - -FROM alpine:3.12 - -ENV UID=1000 \ - GID=1000 \ - USER=youtube - -RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID - -RUN apk add --no-cache \ - ffmpeg \ - npm \ - python2 \ - su-exec \ - && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \ - atomicparsley +# Install backend deps +FROM base as backend WORKDIR /app -COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ] -RUN npm install && chown -R $UID:$GID ./ +COPY [ "backend/","/app/" ] +RUN npm config set strict-ssl false && \ + npm install --prod && \ + ls -al +FROM base as python +WORKDIR /app +COPY docker-utils/GetTwitchDownloader.py . +RUN apt update && \ + apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* +RUN pip install PyGithub requests +RUN python GetTwitchDownloader.py + +# Final image +FROM base +RUN npm install -g pm2 && \ + apt update && \ + apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* +RUN pip install pycryptodomex +WORKDIR /app +# User 1000 already exist from base image +COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ] +COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ] +COPY --chown=$UID:$GID --from=backend ["/app/","/app/"] COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] -COPY --chown=$UID:$GID [ "/backend/", "/app/" ] +RUN chown $UID:$GID . +RUN chmod +x /app/fix-scripts/*.sh +# Add some persistence data +#VOLUME ["/app/appdata"] EXPOSE 17442 ENTRYPOINT [ "/app/entrypoint.sh" ] -CMD [ "node", "app.js" ] +CMD [ "npm","start" ] diff --git a/Dockerfile.heroku b/Dockerfile.heroku new file mode 100644 index 0000000..977a446 --- /dev/null +++ b/Dockerfile.heroku @@ -0,0 +1,2 @@ +FROM tzahi12345/youtubedl-material:latest +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/Procfile b/Procfile deleted file mode 100644 index dc14c0b..0000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: npm start --prefix backend \ No newline at end of file diff --git a/Public API v1.yaml b/Public API v1.yaml index b57c386..83c742e 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -3,41 +3,20 @@ info: title: YoutubeDL-Material API Docs - Official version: '1.0' description: "Welcome to the official docs for YoutubeDL-Material.\n\n\nYou can check out all the available endpoints. Remember to authenticate with your API key using the \"apiKey\" query parameter with your requests.\n\n\nTo do so, simply add this to the end of your API call:\n\n\n`?apiKey=API_KEY`\n\n\nReplce API_KEY with the API key in the settings menu. If one doesn't exist, click generate to create one.\n\n### Multi-user mode\n\nWhen using multi-user mode, you will need to supply a JWT token to authenticate requests through that user. This lets the server know which user to run the task for, like downloading a video for a specific user. \n\nTo do this, you must use the `/api/auth/login` endpoint to login using a user's username and password. This will result in an object containing a `token`. Supply this along with your API key like so:\n\n`?apiKey=API_KEY&jwt=JWT_TOKEN`\n\nNotice the `&` between the `API_KEY` and `jwt`." + contact: + name: Isaac Abadi + url: https://github.com/Tzahi12345/YoutubeDL-Material + email: IsaacMGrynsztein@gmail.com servers: - url: 'http://localhost:17442' paths: - /api/tomp3: - post: - tags: - - downloader - summary: Download audio file - description: |- - Downloads an audio file with the given URL. Will include global args if they exist. - - - HTTP requests will return once the audio file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status. - operationId: post-tomp3 - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200' - security: - - Auth query parameter: [] - /api/tomp4: + /api/downloadFile: post: tags: - downloader summary: Download video file description: |- - Downloads a video file with the given URL. Will include global args if they exist. + Downloads a file with the given URL. Will include global args if they exist. HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status. @@ -46,14 +25,37 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_1' + $ref: '#/components/schemas/DownloadRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_1' + $ref: '#/components/schemas/DownloadResponse' + '500': + description: Server download error + security: + - Auth query parameter: [] + /api/generateArgs: + post: + tags: + - downloader + summary: Generates arguments used to download file + description: Generates args, used for checking what args would run if you ran downloadFile + operationId: post-generateArgs + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GenerateArgsResponse' security: - Auth query parameter: [] /api/getMp3s: @@ -69,8 +71,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_2' - requestBody: {} + $ref: '#/components/schemas/GetMp3sResponse' security: - Auth query parameter: [] /api/getMp4s: @@ -86,7 +87,59 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_3' + $ref: '#/components/schemas/GetMp4sResponse' + security: + - Auth query parameter: [] + /api/getAllFiles: + post: + tags: + - files + summary: Get all files + description: Gets all files and playlists stored in the db + operationId: get-getAllFiles + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllFilesRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllFilesResponse' + security: + - Auth query parameter: [] + /api/rss: + get: + tags: + - files + summary: Generates an RSS feed + description: Generates an RSS feed for downloaded files + operationId: get-rss + parameters: + - in: query + name: params + schema: + allOf: + - $ref: '#/components/schemas/GetAllFilesRequest' + - type: object + properties: + uuid: + type: string + description: user uid + default: null + style: form + explode: true + responses: + '200': + description: OK + content: + text/plain: + schema: + type: string + description: RSS feed security: - Auth query parameter: [] /api/getFile: @@ -100,14 +153,37 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_2' + $ref: '#/components/schemas/GetFileRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_4' + $ref: '#/components/schemas/GetFileResponse' + '401': + description: User is not authorized to view the file. + security: + - Auth query parameter: [] + /api/updateFile: + post: + tags: + - files + summary: Updates file database object + description: Updates a file db object using its uid and a change object. + operationId: post-updateFile + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateFileRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/enableSharing: @@ -122,14 +198,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_3' + $ref: '#/components/schemas/SharingToggle' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/disableSharing: @@ -144,14 +220,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_4' + $ref: '#/components/schemas/SharingToggle' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/incrementViewCount: + post: + summary: Increments a file's view count + tags: + - files + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/IncrementViewCountRequest' + operationId: post-api-incrementViewCount + description: Increments a file's view count security: - Auth query parameter: [] /api/subscribe: @@ -165,14 +262,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_5' + $ref: '#/components/schemas/SubscribeRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_6' + $ref: '#/components/schemas/SubscribeResponse' security: - Auth query parameter: [] /api/unsubscribe: @@ -186,14 +283,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_6' + $ref: '#/components/schemas/UnsubscribeRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_7' + $ref: '#/components/schemas/UnsubscribeResponse' security: - Auth query parameter: [] /api/deleteSubscriptionFile: @@ -207,14 +304,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_7' + $ref: '#/components/schemas/DeleteSubscriptionFileRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_8' + $ref: '#/components/schemas/SuccessObject' '500': description: Internal Server Error security: @@ -230,14 +327,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_8' + $ref: '#/components/schemas/GetSubscriptionRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_9' + $ref: '#/components/schemas/GetSubscriptionResponse' security: - Auth query parameter: [] /api/downloadVideosForSubscription: @@ -251,14 +348,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_9' + $ref: '#/components/schemas/DownloadVideosForSubscriptionRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_10' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/getSubscriptions: @@ -278,7 +375,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_11' + $ref: '#/components/schemas/GetAllSubscriptionsResponse' security: - Auth query parameter: [] /api/createPlaylist: @@ -292,14 +389,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_10' + $ref: '#/components/schemas/CreatePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_12' + $ref: '#/components/schemas/CreatePlaylistResponse' security: - Auth query parameter: [] /api/getPlaylist: @@ -313,35 +410,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_11' + $ref: '#/components/schemas/GetPlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_13' + $ref: '#/components/schemas/GetPlaylistResponse' security: - Auth query parameter: [] /api/updatePlaylist: post: tags: - playlists - summary: Update playlist files - description: Updates the list of filenames in the playlist object + summary: Update playlist + description: Updates the playlist object operationId: post-api-updatePlaylist requestBody: content: application/json: schema: - $ref: '#/components/schemas/body_12' + $ref: '#/components/schemas/UpdatePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/deletePlaylist: @@ -355,14 +452,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_13' + $ref: '#/components/schemas/DeletePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/addFileToPlaylist: + post: + tags: + - playlists + summary: Adds a file to a playlist + description: Adds a file to a playlist + operationId: post-api-addFileToPlaylist + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AddFileToPlaylistRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/deleteMp4: @@ -376,7 +494,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_14' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK @@ -387,7 +505,7 @@ paths: description: Whether the operation succeeded security: - Auth query parameter: [] - /api/downloadFile: + /api/downloadFileFromServer: post: tags: - files @@ -397,32 +515,50 @@ paths: content: application/json: schema: - type: object - properties: {} - application/xml: - schema: - $ref: '#/components/schemas/body_15' + $ref: '#/components/schemas/DownloadFileRequest' responses: '200': description: 'The file itself is in the response, as well as an options object.' + '401': + description: User is not authorized to view the file. security: - Auth query parameter: [] /api/deleteFile: post: tags: - files - summary: Delete downloaded file (unused) + summary: Delete downloaded file operationId: post-api-deleteFile requestBody: content: application/json: schema: - $ref: '#/components/schemas/body_16' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK security: - Auth query parameter: [] + /api/deleteAllFiles: + post: + tags: + - files + summary: Delete all downloaded files + operationId: post-api-deleteAllFiles + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteMp3Mp4Request' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteAllFilesResponse' + security: + - Auth query parameter: [] /api/downloadArchive: post: tags: @@ -434,10 +570,75 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_17' + $ref: '#/components/schemas/DownloadArchiveRequest' responses: '200': description: The archive text file is sent as a response + '404': + description: If the archive dir is not found, 404 is sent as a response + security: + - Auth query parameter: [] + /api/deleteArchiveItems: + post: + tags: + - archive + summary: Delete item from archive + description: 'Deletes an item from the archive' + operationId: post-api-deleteArchiveItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteArchiveItemsRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/importArchive: + post: + tags: + - archive + summary: Imports archive + description: 'Imports an existing archive.txt file' + operationId: post-api-importArchive + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ImportArchiveRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/uploadCookies: + post: + tags: + - downloader + summary: Upload cookies + description: 'Uploads cookies file to be used during downloading' + operationId: post-api-uploadCookies + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadCookiesRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/updaterStatus: @@ -453,7 +654,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_14' + $ref: '#/components/schemas/UpdaterStatus' security: - Auth query parameter: [] /api/updateServer: @@ -467,14 +668,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_18' + $ref: '#/components/schemas/UpdateServerRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/isPinSet: @@ -493,48 +694,6 @@ paths: $ref: '#/components/schemas/inline_response_200_15' security: - Auth query parameter: [] - /api/checkPin: - post: - tags: - - security - summary: Check if pin is correct - description: Checks the pin against an inputted one. Will return true if they match - operationId: post-api-checkPin - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body_19' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200_5' - security: - - Auth query parameter: [] - /api/setPin: - post: - tags: - - security - summary: Set pin - operationId: post-api-setPin - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body_20' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200_5' - security: - - Auth query parameter: [] - description: '' /api/generateNewAPIKey: post: tags: @@ -548,7 +707,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_16' + $ref: '#/components/schemas/GenerateNewApiKeyResponse' security: - Auth query parameter: [] /api/deleteMp3: @@ -562,7 +721,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_21' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK @@ -585,7 +744,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_17' + $ref: '#/components/schemas/ConfigResponse' security: - Auth query parameter: [] /api/setConfig: @@ -597,18 +756,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_22' + $ref: '#/components/schemas/SetConfigRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + '404': + description: Tried to save invalid config file. security: - Auth query parameter: [] /api/downloads: - get: + post: summary: Get info for all downloads tags: - downloader @@ -618,34 +779,13 @@ paths: content: application/json: schema: - type: object - properties: - downloads: - type: object - properties: - uid: - type: string - downloading: - type: boolean - complete: - type: boolean - url: - type: string - type: - type: string - percent_complete: - type: string - is_playlist: - type: boolean - timestamp_start: - type: number - timestamp_end: - type: number - fileNames: - type: array - items: - type: string - operationId: get-api-downloads + $ref: '#/components/schemas/GetAllDownloadsResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllDownloadsRequest' + operationId: post-api-downloads description: Retrieves all downloads recorded by the server and their status. security: - Auth query parameter: [] @@ -659,52 +799,376 @@ paths: content: application/json: schema: - type: object - properties: - download: - type: object - properties: - uid: - type: string - downloading: - type: boolean - complete: - type: boolean - url: - type: string - type: - type: string - percent_complete: - type: string - is_playlist: - type: boolean - timestamp_start: - type: number - timestamp_end: - type: number - fileNames: - type: array - items: - type: string + $ref: '#/components/schemas/GetDownloadResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Gets a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/pauseDownload: + post: + summary: Pauses one download + operationId: post-api-pause-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Pause a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/pauseAllDownloads: + post: + tags: + - downloader + summary: Pauses all downloads + operationId: post-api-pause-all-downloads requestBody: content: application/json: schema: type: object - properties: - session_id: - type: string - download_id: - type: string - required: - - session_id - - download_id + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/resumeDownload: + post: + summary: Resume one download + operationId: post-api-resume-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' description: '' - description: "Gets a single download using its download_id and session_id. session_id is the device fingerprint. If none was provided at the time of download, then set session_id is 'undeclared'." + description: "Resume a single download using its download_id." security: - Auth query parameter: [] tags: - downloader + /api/resumeAllDownloads: + post: + tags: + - downloader + summary: Resumes all downloads + operationId: post-api-resume-all-downloads + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/restartDownload: + post: + summary: Restart one download + operationId: post-api-restart-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/RestartDownloadResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Restart a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/cancelDownload: + post: + summary: Cancel one download + operationId: post-api-cancel-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Cancel a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/clearDownload: + post: + summary: Clear one download + operationId: post-api-clear-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Clears a single download from the downloaded list using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/clearDownloads: + post: + summary: Clear multiple downloads + operationId: post-api-clear-downloads + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ClearDownloadsRequest' + description: '' + description: "Clears multiple downloads based on a given filter." + security: + - Auth query parameter: [] + tags: + - downloader + /api/getTask: + post: + summary: Get info for one task + operationId: post-api-get-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + description: '' + description: "Gets a single task using its key." + security: + - Auth query parameter: [] + tags: + - tasks + /api/getTasks: + post: + tags: + - tasks + summary: Get tasks + operationId: post-api-get-tasks + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllTasksResponse' + /api/resetTasks: + post: + tags: + - tasks + summary: Resets all tasks + operationId: post-api-reset-tasks + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/runTask: + post: + tags: + - tasks + summary: Runs one task + operationId: post-api-run-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/confirmTask: + post: + tags: + - tasks + summary: Confirms a task + operationId: post-api-confirm-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/cancelTask: + post: + tags: + - tasks + summary: Cancels a task + operationId: post-api-cancel-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/updateTaskSchedule: + post: + tags: + - tasks + summary: Updates task schedule + operationId: post-api-update-task-schedule + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateTaskScheduleRequest' + /api/updateTaskData: + post: + tags: + - tasks + summary: Updates task data + operationId: post-api-update-task-data + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateTaskDataRequest' + /api/getDBBackups: + post: + tags: + - tasks + summary: Get database backups + operationId: post-api-get-database-backups + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetDBBackupsResponse' + requestBody: + content: + application/json: + schema: + type: object + /api/restoreDBBackup: + post: + tags: + - tasks + summary: Restore database backup + operationId: post-api-restore-database-backup + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RestoreDBBackupRequest' /api/auth/login: post: summary: Login @@ -715,35 +1179,13 @@ paths: content: application/json: schema: - type: object - properties: - user: - $ref: '#/components/schemas/user' - token: - type: string - permissions: - type: array - items: - type: string - available_permissions: - type: array - items: - type: string + $ref: '#/components/schemas/LoginResponse' description: Use this method to log into a user using their username and password and receive a jwt auth token so you can send per-user requests. requestBody: content: application/json: schema: - type: object - properties: - userid: - type: string - description: 'This is the username, not the user uid' - password: - type: string - required: - - userid - - password + $ref: '#/components/schemas/LoginRequest' security: - Auth query parameter: [] tags: @@ -758,27 +1200,13 @@ paths: content: application/json: schema: - type: object - properties: - user: - $ref: '#/components/schemas/user' + $ref: '#/components/schemas/RegisterResponse' description: Use this endpoint to register a user. It will only work if registration is enabled. requestBody: content: application/json: schema: - type: object - properties: - userid: - type: string - username: - type: string - password: - type: string - required: - - userid - - username - - password + $ref: '#/components/schemas/RegisterRequest' security: - Auth query parameter: [] tags: @@ -790,21 +1218,15 @@ paths: responses: '200': description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' requestBody: content: application/json: schema: - type: object - properties: - change_object: - type: object - properties: - name: - type: string - role: - type: string - required: - - change_object + $ref: '#/components/schemas/UpdateUserRequest' description: Updates certain properties for a user. Only two are possible right now. security: - Auth query parameter: [] @@ -820,21 +1242,13 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' description: Deletes a user by its uid. requestBody: content: application/json: schema: - type: object - properties: - uid: - type: string - required: - - uid + $ref: '#/components/schemas/DeleteUserRequest' security: - Auth query parameter: [] tags: @@ -849,25 +1263,7 @@ paths: content: application/json: schema: - type: object - properties: - roles: - type: object - properties: - admin: - type: object - properties: - permissions: - type: array - items: - type: string - user: - type: object - properties: - permissions: - type: array - items: - type: string + $ref: '#/components/schemas/GetRolesResponse' description: Gets the available roles and their permissions security: - Auth query parameter: [] @@ -883,10 +1279,7 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' description: "Changes the permissions for a user. Available values for each permission are: `default`, `yes`, and `no`. `default` will use the user's role's default permission." security: - Auth query parameter: [] @@ -894,18 +1287,7 @@ paths: content: application/json: schema: - type: object - properties: - user_uid: - type: string - permission: - type: string - new_value: - type: string - required: - - user_uid - - permission - - new_value + $ref: '#/components/schemas/ChangeUserPermissionsRequest' tags: - multi-user mode /api/changeRolePermissions: @@ -918,26 +1300,12 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' requestBody: content: application/json: schema: - type: object - properties: - role: - type: string - permission: - type: string - new_value: - type: string - required: - - role - - permission - - new_value + $ref: '#/components/schemas/ChangeRolePermissionsRequest' description: 'Changes the permissions for a role. Available values for each permission are: `yes`, and `no`.' security: - Auth query parameter: [] @@ -953,20 +1321,339 @@ paths: content: application/json: schema: - type: object - properties: - users: - type: array - items: - $ref: '#/components/schemas/user' + $ref: '#/components/schemas/GetUsersResponse' description: 'Gets all users, returns a list of the user objects including their user permissions, videos, playlists, subscriptions, etc.' security: - Auth query parameter: [] tags: - multi-user mode + /api/versionInfo: + get: + tags: + - server + summary: Gets server version info + operationId: get-api-versionInfo + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/VersionInfoResponse' + security: + - Auth query parameter: [] + /api/getLogs: + post: + summary: Gets logs from server + tags: + - server + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetLogsResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetLogsRequest' + operationId: post-api-getLogs + description: Gets logs from server + security: + - Auth query parameter: [] + /api/clearAllLogs: + post: + summary: Clears the log file + tags: + - server + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + operationId: post-api-clearAllLogs + description: Clears the log file + security: + - Auth query parameter: [] + /api/getDBInfo: + get: + tags: + - db + summary: Gets information on the DB + operationId: get-api-getDBInfo + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/DBInfoResponse' + security: + - Auth query parameter: [] + /api/transferDB: + post: + summary: Transfers DB between Local and MongoDB + tags: + - db + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TransferDBResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TransferDBRequest' + operationId: post-api-transferDB + description: Initiates a transfer between Local and MongoDB. Connection string must be set. + security: + - Auth query parameter: [] + /api/testConnectionString: + post: + summary: Tests a MongoDB connection string + tags: + - db + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TestConnectionStringResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TestConnectionStringRequest' + operationId: post-api-testConnectionString + description: Tests a MongoDB connection string and returns an error if one exists. + security: + - Auth query parameter: [] + /api/getFullTwitchChat: + post: + summary: Gets the downloaded Twitch Chat (VODs only) + tags: + - twitch + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetFullTwitchChatResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetFullTwitchChatRequest' + operationId: post-api-getFullTwitchChat + description: Gets the downloaded Twitch Chat (VODs only) + security: + - Auth query parameter: [] + /api/downloadTwitchChatByVODID: + post: + summary: Downloads Twitch Chat for a VOD + tags: + - twitch + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadTwitchChatByVODIDResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadTwitchChatByVODIDRequest' + operationId: post-api-downloadTwitchChatByVODID + description: Downloads Twitch Chat for a VOD + security: + - Auth query parameter: [] + /api/checkConcurrentStream: + post: + summary: Checks status of a concurrent stream + tags: + - player + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/CheckConcurrentStreamResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CheckConcurrentStreamRequest' + operationId: post-api-checkConcurrentStream + description: Checks status of a concurrent stream + security: + - Auth query parameter: [] + /api/updateConcurrentStream: + post: + summary: Updates a concurrent stream + tags: + - player + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateConcurrentStreamResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateConcurrentStreamRequest' + operationId: post-api-updateConcurrentStream + description: Updates a concurrent stream + security: + - Auth query parameter: [] + /api/getAllCategories: + post: + summary: Gets all categories + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllCategoriesResponse' + operationId: post-api-getAllCategories + description: Gets all categories + security: + - Auth query parameter: [] + /api/createCategory: + post: + summary: Creates a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCategoryResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCategoryRequest' + operationId: post-api-createCategory + description: Creates a category + security: + - Auth query parameter: [] + /api/deleteCategory: + post: + summary: Deletes a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteCategoryRequest' + operationId: post-api-deleteCategory + description: Deletes a category + security: + - Auth query parameter: [] + /api/updateCategory: + post: + summary: Updates a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateCategoryRequest' + operationId: post-api-updateCategory + description: Updates a category + security: + - Auth query parameter: [] + /api/updateCategories: + post: + summary: Updates all categories + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateCategoriesRequest' + operationId: post-api-updateCategories + description: Updates all categories + security: + - Auth query parameter: [] components: schemas: - body: + SuccessObject: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + FileType: + type: string + enum: + - audio + - video + CropFileSettings: + type: object + required: + - cropFileStart + - cropFileEnd + properties: + cropFileStart: + type: number + cropFileEnd: + type: number + Config: + required: + - YoutubeDLMaterial + type: object + properties: + YoutubeDLMaterial: + type: object + DownloadRequest: required: - url type: object @@ -975,64 +1662,162 @@ components: type: string customQualityConfiguration: type: string + description: Video format code. Overrides other quality options. example: '251' - maxBitrate: - type: string - example: '160' customArgs: type: string + description: Custom command-line arguments for youtube-dl. Overrides all other options, except url. + additionalArgs: + type: string + description: Additional command-line arguments for youtube-dl. Added to whatever args would normally be used. customOutput: type: string + description: Custom output filename template. youtubeUsername: type: string + description: Login with this account ID youtubePassword: type: string - inline_response_200: - required: - - audiopathEncoded - - uid - type: object - properties: - uid: - type: string - file_names: - type: string - audiopathEncoded: - type: string - body_1: - required: - - url - type: object - properties: - url: - type: string - customQualityConfiguration: - type: string - example: 242+251 + description: Account password selectedHeight: type: string + description: Height of the video, if known example: '1080' - customArgs: + maxHeight: type: string - customOutput: + description: Max height that should be used, useful for playlists. selectedHeight will override this. + example: '1080' + maxBitrate: type: string - youtubeUsername: - type: string - youtubePassword: - type: string - inline_response_200_1: - required: - - uid - - videopathEncoded + description: Specify ffmpeg/avconv audio quality + example: '160' + type: + $ref: '#/components/schemas/FileType' + cropFileSettings: + $ref: '#/components/schemas/CropFileSettings' + ignoreArchive: + type: boolean + description: If using youtube-dl archive, download will ignore it + DownloadResponse: type: object properties: - uid: + download: + $ref: '#/components/schemas/Download' + GenerateArgsResponse: + type: object + properties: + args: + type: array + items: + type: string + GetDownloadRequest: + type: object + properties: + download_uid: type: string - file_names: + required: + - download_uid + GetDownloadResponse: + type: object + properties: + download: + $ref: '#/components/schemas/Download' + RestartDownloadResponse: + allOf: + - $ref: '#/components/schemas/SuccessObject' + - type: object + properties: + new_download_uid: + type: string + GetAllDownloadsRequest: + type: object + properties: + uids: + type: array + items: + type: string + description: Filters downloads with the array + nullable: true + GetAllDownloadsResponse: + type: object + properties: + downloads: + type: array + items: + $ref: '#/components/schemas/Download' + ClearDownloadsRequest: + type: object + properties: + clear_finished: + type: boolean + clear_paused: + type: boolean + clear_errors: + type: boolean + GetTaskRequest: + type: object + properties: + task_key: type: string - videopathEncoded: + required: + - task_key + UpdateTaskScheduleRequest: + type: object + properties: + task_key: type: string - inline_response_200_2: + new_schedule: + $ref: '#/components/schemas/Schedule' + required: + - task_key + - new_schedule + UpdateTaskDataRequest: + type: object + properties: + task_key: + type: string + new_data: + type: object + required: + - task_key + - new_data + UpdateTaskOptionsRequest: + type: object + properties: + task_key: + type: string + new_options: + type: object + required: + - task_key + - new_options + GetTaskResponse: + type: object + properties: + task: + $ref: '#/components/schemas/Task' + GetAllTasksResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/Task' + GetDBBackupsResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/DBBackup' + RestoreDBBackupRequest: + type: object + required: + - file_name + properties: + file_name: + type: string + GetMp3sResponse: required: - mp3s - playlists @@ -1041,80 +1826,133 @@ components: mp3s: type: array items: - $ref: '#/components/schemas/inline_response_200_2_mp3s' + $ref: '#/components/schemas/DatabaseFile' playlists: type: array description: All audio playlists items: - $ref: '#/components/schemas/inline_response_200_2_playlists' - inline_response_200_3: + $ref: '#/components/schemas/Playlist' + GetMp4sResponse: required: - mp4s + - playlists type: object properties: mp4s: type: array items: - $ref: '#/components/schemas/inline_response_200_3_mp4s' + $ref: '#/components/schemas/DatabaseFile' playlists: type: array description: All video playlists items: - type: object - body_2: + $ref: '#/components/schemas/Playlist' + GetAllFilesRequest: + type: object + properties: + sort: + $ref: '#/components/schemas/Sort' + range: + type: array + items: + type: number + description: Two elements allowed, start index and end index + minItems: 2 + maxItems: 2 + default: null + text_search: + type: string + description: Filter files by title + default: null + file_type_filter: + $ref: '#/components/schemas/FileTypeFilter' + favorite_filter: + type: boolean + description: If set to true, only gets favorites + default: false + sub_id: + type: string + description: Include if you want to filter by subscription + default: null + Sort: + type: object + properties: + by: + type: string + description: Property to sort by + default: registered + order: + type: number + description: 1 for ascending, -1 for descending + default: -1 + FileTypeFilter: + type: string + enum: + - audio_only + - video_only + - both + default: both + GetAllFilesResponse: + required: + - files + - playlists + type: object + properties: + files: + type: array + items: + $ref: '#/components/schemas/DatabaseFile' + playlists: + type: array + description: All video playlists + items: + $ref: '#/components/schemas/Playlist' + GetFileRequest: required: - uid type: object properties: uid: type: string + description: Video UID type: + $ref: '#/components/schemas/FileType' + uuid: type: string - inline_response_200_4: + description: User UID + GetFileResponse: required: - - file - success type: object properties: success: - type: string + type: boolean file: - $ref: '#/components/schemas/inline_response_200_2_mp3s' - body_3: + $ref: '#/components/schemas/DatabaseFile' + UpdateFileRequest: + required: + - uid + - change_obj + type: object + properties: + uid: + type: string + description: Video UID + change_obj: + type: object + description: Object with fields to update as keys and their new values + SharingToggle: required: - - is_playlist - - type - uid type: object properties: uid: type: string - type: - type: string is_playlist: type: boolean - inline_response_200_5: - required: - - success - type: object - properties: - success: - type: boolean - body_4: - required: - - type - - uid - type: object - properties: - type: - type: string - uid: - type: string - description: uid is either the video uid or the playlist ID - is_playlist: - type: boolean - body_5: + SubscribeRequest: required: + - name - url type: object properties: @@ -1124,48 +1962,34 @@ components: type: string timerange: type: string - streamingOnly: + audioOnly: type: boolean - inline_response_200_6: + customArgs: + type: string + customFileOutput: + type: string + maxQuality: + type: string + SubscribeResponse: required: - new_sub type: object properties: new_sub: - $ref: '#/components/schemas/inline_response_200_6_new_sub' + $ref: '#/components/schemas/Subscription' error: type: string - apiunsubscribe_sub: - required: - - id - - name - - url - - videos - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - videos: - type: array - items: - type: object - body_6: + UnsubscribeRequest: required: - sub type: object properties: sub: - $ref: '#/components/schemas/apiunsubscribe_sub' + $ref: '#/components/schemas/SubscriptionRequestData' deleteMode: type: boolean description: Defaults to false - inline_response_200_7: + UnsubscribeResponse: required: - success type: object @@ -1174,81 +1998,56 @@ components: type: boolean error: type: string - apideleteSubscriptionFile_sub: - required: - - id - - isPlaylist - - name - - url - - videos + DeleteAllFilesResponse: type: object properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - videos: - type: array - items: - type: object - body_7: + file_count: + type: number + description: Number of files found matching search parameters + delete_count: + type: number + description: Number of files removed + DeleteSubscriptionFileRequest: required: - - file - - sub + - file_uid type: object properties: - file: + file_uid: type: string - sub: - $ref: '#/components/schemas/apideleteSubscriptionFile_sub' deleteForever: type: boolean description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.' - inline_response_200_8: - type: object - properties: - success: - type: boolean - body_8: + GetSubscriptionRequest: required: - - subID + - id type: object properties: - subID: + id: type: string - inline_response_200_9: + description: Subscription ID + name: + type: string + description: Subscription name + GetSubscriptionResponse: required: - files - subscription type: object properties: subscription: - $ref: '#/components/schemas/inline_response_200_9_subscription' + $ref: '#/components/schemas/Subscription' files: type: array items: type: object - body_9: + DownloadVideosForSubscriptionRequest: required: - subID type: object properties: subID: type: string - inline_response_200_10: - required: - - success - type: object - properties: - success: - type: number - inline_response_200_11: + GetAllSubscriptionsResponse: required: - subscriptions type: object @@ -1256,140 +2055,182 @@ components: subscriptions: type: array items: - $ref: '#/components/schemas/inline_response_200_11_subscriptions' - body_10: + $ref: '#/components/schemas/Subscription' + CreatePlaylistRequest: required: - - fileNames + - uids - playlistName - thumbnailURL - - type type: object properties: playlistName: type: string - fileNames: + uids: type: array items: type: string - type: - type: string thumbnailURL: type: string - inline_response_200_12: + CreatePlaylistResponse: required: - new_playlist - success type: object properties: new_playlist: - $ref: '#/components/schemas/inline_response_200_12_new_playlist' + $ref: '#/components/schemas/Playlist' success: type: boolean - body_11: + GetPlaylistRequest: required: - - playlistID - - type + - playlist_id type: object properties: - playlistID: + playlist_id: type: string type: + $ref: '#/components/schemas/FileType' + uuid: type: string - inline_response_200_13: + include_file_metadata: + type: boolean + GetPlaylistResponse: required: - playlist - success - - type type: object properties: playlist: - $ref: '#/components/schemas/inline_response_200_2_playlists' - type: - type: string + $ref: '#/components/schemas/Playlist' success: type: boolean - body_12: - required: - - fileNames - - playlistID - - type + file_objs: + type: array + description: File objects for every uid in the playlist's uids property, in the same order + items: + $ref: '#/components/schemas/DatabaseFile' + GetPlaylistsRequest: type: object properties: - playlistID: - type: string - fileNames: + include_categories: + type: boolean + GetPlaylistsResponse: + required: + - playlists + type: object + properties: + playlists: type: array items: - type: string - type: - type: string - body_13: + $ref: '#/components/schemas/Playlist' + UpdatePlaylistRequest: required: - - playlistID - - type + - playlist type: object properties: - playlistID: - type: string - type: - type: string - body_14: + playlist: + $ref: '#/components/schemas/Playlist' + DeletePlaylistRequest: required: - - uid + - playlist_id + type: object + properties: + playlist_id: + type: string + DownloadFileRequest: type: object properties: uid: type: string - body_15: - required: - - fileNames - - type + uuid: + type: string + sub_id: + type: string + playlist_id: + type: string + url: + type: string + type: + $ref: '#/components/schemas/FileType' + DownloadArchiveRequest: type: object properties: - fileNames: + type: + $ref: '#/components/schemas/FileType' + sub_id: + type: string + Archive: + required: + - extractor + - id + - type + - title + - timestamp + - uid + type: object + properties: + extractor: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + title: + type: string + user_uid: + type: string + sub_id: + type: string + timestamp: + type: number + uid: + type: string + DeleteArchiveItemsRequest: + type: object + required: + - archives + properties: + archives: type: array - description: Array of 1 or more files to download items: - type: string - zip_mode: - type: boolean - type: - type: string - outputName: - type: string - fullPathProvided: - type: string - subscriptionName: - type: boolean - description: Only used for subscriptions - subscriptionPlaylist: - type: boolean - description: Only used for subscriptions - body_16: + $ref: '#/components/schemas/Archive' + ImportArchiveRequest: + type: object required: - - fileName + - archive - type - type: object properties: - fileName: + archive: type: string type: + $ref: '#/components/schemas/FileType' + sub_id: type: string - apidownloadArchive_sub: - required: - - archive_dir + GetArchivesRequest: type: object properties: - archive_dir: + type: + $ref: '#/components/schemas/FileType' + sub_id: type: string - body_17: - required: - - sub + GetArchivesResponse: type: object + required: + - archives properties: - sub: - $ref: '#/components/schemas/apidownloadArchive_sub' - inline_response_200_14: + archives: + type: array + items: + $ref: '#/components/schemas/Archive' + UploadCookiesRequest: + type: object + required: + - cookies + properties: + cookies: + type: string + format: binary + UpdaterStatus: required: - details - updating @@ -1399,11 +2240,270 @@ components: type: boolean details: type: string - body_18: + error: + type: boolean + UpdateServerRequest: + required: + - tag type: object properties: tag: type: string + DBInfoResponse: + type: object + properties: + using_local_db: + type: boolean + stats_by_table: + type: object + properties: + files: + $ref: '#/components/schemas/TableInfo' + playlists: + $ref: '#/components/schemas/TableInfo' + categories: + $ref: '#/components/schemas/TableInfo' + subscriptions: + $ref: '#/components/schemas/TableInfo' + users: + $ref: '#/components/schemas/TableInfo' + roles: + $ref: '#/components/schemas/TableInfo' + download_queue: + $ref: '#/components/schemas/TableInfo' + archives: + $ref: '#/components/schemas/TableInfo' + TransferDBResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + TransferDBRequest: + required: + - local_to_remote + type: object + properties: + local_to_remote: + description: True if transfering DB from Local to MongoDB, false if transferring DB from MongoDB to Local + type: boolean + TestConnectionStringResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + TestConnectionStringRequest: + required: + - connection_string + type: object + properties: + connection_string: + description: MongoDB connection string + type: string + GetFullTwitchChatResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + GetFullTwitchChatRequest: + required: + - id + - type + type: object + properties: + id: + description: File ID + type: string + type: + $ref: '#/components/schemas/FileType' + uuid: + description: User UID + type: string + sub: + $ref: '#/components/schemas/Subscription' + DownloadTwitchChatByVODIDResponse: + required: + - chat + type: object + properties: + chat: + type: array + items: + $ref: '#/components/schemas/TwitchChatMessage' + DownloadTwitchChatByVODIDRequest: + required: + - id + - type + - vodId + type: object + properties: + id: + description: File ID + type: string + vodId: + description: ID of the VOD + type: string + type: + $ref: '#/components/schemas/FileType' + uuid: + description: User UID + type: string + sub: + $ref: '#/components/schemas/Subscription' + CheckConcurrentStreamResponse: + required: + - stream + type: object + properties: + stream: + $ref: '#/components/schemas/ConcurrentStream' + CheckConcurrentStreamRequest: + required: + - uid + type: object + properties: + uid: + description: UID of the concurrent stream + type: string + UpdateConcurrentStreamResponse: + required: + - stream + type: object + properties: + stream: + $ref: '#/components/schemas/ConcurrentStream' + UpdateConcurrentStreamRequest: + allOf: + - $ref: '#/components/schemas/ConcurrentStream' + - type: object + properties: + uid: + type: string + description: Concurrent stream UID + required: + - uid + GetLogsResponse: + type: object + properties: + logs: + type: string + description: Number of lines to retrieve from the bottom + success: + type: boolean + GetLogsRequest: + required: + - connection_string + type: object + properties: + lines: + type: number + GetFileFormatsRequest: + type: object + properties: + url: + type: string + GetFileFormatsResponse: + required: + - result + - success + type: object + properties: + success: + type: boolean + result: + allOf: + - type: object + properties: + formats: + type: array + items: + type: object + IncrementViewCountRequest: + required: + - file_uid + type: object + properties: + file_uid: + type: string + sub_id: + type: string + uuid: + type: string + description: User UID + AddFileToPlaylistRequest: + required: + - playlist_id + - file_uid + type: object + properties: + file_uid: + type: string + playlist_id: + type: string + GetAllCategoriesResponse: + required: + - categories + type: object + properties: + categories: + type: array + items: + $ref: '#/components/schemas/Category' + CreateCategoryResponse: + type: object + properties: + new_category: + $ref: '#/components/schemas/Category' + success: + type: boolean + CreateCategoryRequest: + required: + - name + type: object + properties: + name: + type: string + DeleteCategoryRequest: + required: + - category_uid + type: object + properties: + category_uid: + type: string + UpdateCategoryRequest: + required: + - category + type: object + properties: + category: + $ref: '#/components/schemas/Category' + UpdateCategoriesRequest: + required: + - categories + type: object + properties: + categories: + type: array + items: + $ref: '#/components/schemas/Category' + VersionInfoResponse: + required: + - version_info + type: object + properties: + version_info: + $ref: '#/components/schemas/Version' inline_response_200_15: required: - is_set @@ -1425,7 +2525,7 @@ components: properties: unhashed_pin: type: string - inline_response_200_16: + GenerateNewApiKeyResponse: required: - new_api_key type: object @@ -1433,31 +2533,33 @@ components: new_api_key: type: string example: 4241b401-7236-493e-92b5-b72696b9d853 - body_21: + DeleteMp3Mp4Request: required: - uid type: object properties: uid: type: string - inline_response_200_17: + blacklistMode: + type: boolean + ConfigResponse: required: - config_file - success type: object properties: config_file: - type: object + $ref: '#/components/schemas/Config' success: type: boolean - body_22: + SetConfigRequest: required: - new_config_file type: object properties: new_config_file: - type: object - inline_response_200_2_mp3s: + $ref: '#/components/schemas/Config' + DatabaseFile: required: - duration - id @@ -1470,6 +2572,7 @@ components: - upload_date - uploader - url + - favorite type: object properties: id: @@ -1478,6 +2581,9 @@ components: type: string thumbnailURL: type: string + description: Backup if thumbnailPath is not defined + thumbnailPath: + type: string isAudio: type: boolean duration: @@ -1489,25 +2595,49 @@ components: type: string size: type: number + description: In bytes path: type: string upload_date: type: string uid: type: string + user_uid: + type: string sharingEnabled: type: boolean - inline_response_200_2_playlists: + category: + $ref: '#/components/schemas/Category' + view_count: + type: number + local_view_count: + type: number + sub_id: + type: string + registered: + type: number + height: + type: number + description: In pixels, only for videos + abr: + type: number + description: In Kbps + favorite: + type: boolean + Playlist: required: - - fileNames + - uids - id - name - thumbnailURL + - type + - registered + - duration type: object properties: name: type: string - fileNames: + uids: type: array items: type: string @@ -1515,125 +2645,204 @@ components: type: string thumbnailURL: type: string - inline_response_200_3_mp4s: - required: - - duration - - id - - isAudio - - path - - size - - thumbnailURL - - title - - uid - - upload_date - - uploader - - url - type: object - properties: - id: - type: string - title: - type: string - thumbnailURL: - type: string - isAudio: - type: boolean + type: + $ref: '#/components/schemas/FileType' + registered: + type: number duration: type: number - url: - type: string - uploader: - type: string - size: - type: number - path: - type: string - upload_date: - type: string - uid: + user_uid: type: string + auto: + type: boolean sharingEnabled: type: boolean - inline_response_200_6_new_sub: + Download: required: - - id - - name - url - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - timerange: - type: string - inline_response_200_9_subscription: - required: - - archive - - id - - isPlaylist - - name - - url - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - archive: - type: string - inline_response_200_11_subscriptions: - required: - - archive - - id - - isPlaylist - - name - - streamingOnly - - url - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - archive: - type: string - inline_response_200_12_new_playlist: - required: - - fileNames - - id - - name - - thumbnailURL - type + - title + - options + - uid + - step_index + - paused + - running + - finished_step + - percent_complete + - finished + - timestamp_start type: object properties: - name: + uid: type: string - fileNames: + ui_uid: type: string - id: - type: string - thumbnailURL: + running: + type: boolean + finished: + type: boolean + paused: + type: boolean + finished_step: + type: boolean + url: type: string type: type: string - user: + title: + type: string + step_index: + type: number + percent_complete: + type: number + timestamp_start: + type: number + error: + type: string + description: Error text, set if download fails. + nullable: true + error_type: + type: string + description: Error type, may or may not be set in case of an error + nullable: true + user_uid: + type: string + sub_id: + type: string + sub_name: + type: string + prefetched_info: + type: object + Task: + required: + - key + - last_ran + - last_confirmed + - running + - confirming + - data + - error + - schedule + type: object + properties: + key: + type: string + title: + type: string + last_ran: + type: number + last_confirmed: + type: number + running: + type: boolean + confirming: + type: boolean + data: + type: object + error: + type: string + schedule: + type: object + options: + type: object + Schedule: + required: + - type + - data + type: object + properties: + type: + type: string + enum: + - timestamp + - recurring + data: + type: object + properties: + dayOfWeek: + type: array + items: + type: number + hour: + type: number + minute: + type: number + timestamp: + type: number + tz: + type: string + DBBackup: + required: + - name + - timestamp + - size + - source + type: object + properties: + name: + type: string + timestamp: + type: number + size: + type: number + source: + type: string + enum: + - local + - remote + SubscriptionRequestData: + required: + - id + - name + type: object + properties: + name: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + isPlaylist: + type: boolean + archive: + type: string + Subscription: + required: + - id + - name + - url + - type + - user_uid + - isPlaylist + - videos + type: object + properties: + name: + type: string + url: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + user_uid: + type: string + nullable: true + isPlaylist: + type: boolean + archive: + type: string + timerange: + type: string + custom_args: + type: string + custom_output: + type: string + videos: + type: array + items: + type: object + User: title: user type: object properties: @@ -1643,32 +2852,10 @@ components: type: string passhash: type: string - files: - type: object - properties: - audio: - type: array - items: - $ref: '#/components/schemas/file' - video: - type: array - items: - $ref: '#/components/schemas/file' - playlists: - type: object - properties: - audio: - type: array - items: - $ref: '#/components/schemas/file' - video: - type: array - items: - $ref: '#/components/schemas/file' subscriptions: type: array items: - $ref: '#/components/schemas/inline_response_200_9_subscription' + $ref: '#/components/schemas/Subscription' created: type: number role: @@ -1676,17 +2863,294 @@ components: permissions: type: array items: - type: string + $ref: '#/components/schemas/UserPermission' permission_overrides: type: array items: - type: string + $ref: '#/components/schemas/UserPermission' + UserPermission: + type: string + enum: + - filemanager + - settings + - subscriptions + - sharing + - advanced_download + - downloads_manager + YesNo: + type: string + enum: + - 'yes' + - 'no' + TableInfo: + type: object + properties: + records_count: + type: number + TwitchChatMessage: + type: object + properties: + created_at: + type: string + content_offset_seconds: + type: number + commenter: + type: object + properties: + name: + type: string + _id: + type: string + created_at: + type: string + message: + type: object + properties: + body: + type: string + user_color: + type: string + ConcurrentStream: + type: object + properties: + playback_timestamp: + type: number + unix_timestamp: + type: number + playing: + type: boolean + Category: + type: object + properties: + name: + type: string + uid: + type: string + rules: + type: array + items: + $ref: '#/components/schemas/CategoryRule' + custom_output: + type: string + description: Overrides file output for downloaded files in category + CategoryRule: + type: object + properties: + preceding_operator: + type: string + enum: + - 'or' + - 'and' + comparator: + type: string + enum: + - 'includes' + - 'not_includes' + - 'equals' + - 'not_equals' + Version: + type: object + properties: + type: + type: string + tag: + type: string + commit: + type: string + date: + type: string + Notification: + required: + - uid + - type + - text + - read + - timestamp + type: object + properties: + type: + $ref: '#/components/schemas/NotificationType' + uid: + type: string + user_uid: + type: string + action: + type: array + items: + $ref: '#/components/schemas/NotificationAction' + read: + type: boolean + data: + type: object + timestamp: + type: number + NotificationAction: + type: string + enum: + - play + - retry_download + - view_download_error + - view_tasks + NotificationType: + type: string + enum: + - download_complete + - download_error + - task_finished + BaseChangePermissionsRequest: + required: + - permission + - new_value + type: object + properties: + permission: + $ref: '#/components/schemas/UserPermission' + new_value: + $ref: '#/components/schemas/YesNo' + ChangeUserPermissionsRequest: + allOf: + - $ref: '#/components/schemas/BaseChangePermissionsRequest' + - type: object + properties: + user_uid: + type: string + required: + - user_uid + ChangeRolePermissionsRequest: + allOf: + - $ref: '#/components/schemas/BaseChangePermissionsRequest' + - type: object + properties: + role: + type: string + required: + - role file: title: file type: object properties: id: type: string + RegisterRequest: + required: + - userid + - username + - password + type: object + properties: + userid: + type: string + username: + type: string + password: + type: string + RegisterResponse: + type: object + properties: + user: + $ref: '#/components/schemas/User' + LoginRequest: + required: + - username + - password + type: object + properties: + username: + type: string + password: + type: string + LoginResponse: + type: object + properties: + user: + $ref: '#/components/schemas/User' + token: + type: string + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + available_permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + UpdateUserRequest: + required: + - change_object + type: object + properties: + change_object: + required: + - uid + type: object + properties: + uid: + type: string + name: + type: string + role: + type: string + DeleteUserRequest: + required: + - uid + type: object + properties: + uid: + type: string + GetUsersResponse: + required: + - users + type: object + properties: + users: + type: array + items: + $ref: '#/components/schemas/User' + GetRolesResponse: + required: + - roles + type: object + properties: + roles: + type: object + properties: + admin: + type: object + properties: + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + user: + type: object + properties: + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + DeleteNotificationRequest: + required: + - uid + type: object + properties: + uid: + type: string + SetNotificationsToReadRequest: + required: + - uids + type: object + properties: + uids: + type: array + items: + type: string + GetNotificationsResponse: + type: object + properties: + notifications: + type: array + items: + $ref: '#/components/schemas/Notification' securitySchemes: Auth query parameter: name: apiKey diff --git a/README.md b/README.md index c073161..d81227c 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,15 @@ [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues) [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md) -YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend. +YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend. Now with [Docker](#Docker) support! +
+ ## Getting Started -Check out the prerequisites, and go to the installation section. Easy as pie! +Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie! Here's an image of what it'll look like once you're done: @@ -29,7 +31,7 @@ NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker Debian/Ubuntu: ```bash -sudo apt-get install nodejs youtube-dl ffmpeg +sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm ``` CentOS 7: @@ -46,9 +48,12 @@ sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel Optional dependencies: * AtomicParsley (for embedding thumbnails, package name `atomicparsley`) +* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats) ### Installing +If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue: + 1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)! 2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file. @@ -67,7 +72,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend. -Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. +Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`. @@ -77,6 +82,10 @@ Alternatively, you can port forward the port specified in the config (defaults t ## Docker +### Host-specific instructions + +If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp) + ### Setup If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple. @@ -86,8 +95,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**. 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done! -NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`. - ### Custom UID/GID By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so: @@ -98,6 +105,12 @@ environment: GID: YOUR_GID ``` +## MongoDB + +For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)! + +[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M). + ## API [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml) @@ -106,6 +119,12 @@ To get started, go to the settings menu and enable the public API from the *Extr Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above. +## iOS Shortcut + +If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?) + +You can download Shortcut [here.](https://routinehub.co/shortcut/10283/) + ## Contributing If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away. @@ -124,12 +143,16 @@ Official translators: * German - UnlimitedCookies * Chinese - TyRoyal -See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project. +See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project. ## License This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details +## Legal Disclaimer + +This project is in no way affiliated with Google LLC, Alphabet Inc. or YouTube (or their subsidiaries) nor endorsed by them. + ## Acknowledgments * youtube-dl diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0601c7a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +If you would like to see the latest updates, use the `nightly` tag on Docker. + +If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). + +| Version | Supported | +| -------------------- | ------------------ | +| 4.3 Docker Nightlies | :white_check_mark: | +| 4.3 Release | :white_check_mark: | +| 4.2 Release | :x: | +| < 4.2 | :x: | + +## Reporting a Vulnerability + +Please file an issue in our GitHub's repo, because this app +isn't meant to be safe to run as public instance yet, but rather as a LAN facing app. + +We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now. diff --git a/angular.json b/angular.json index 46e84ea..8f50ad6 100644 --- a/angular.json +++ b/angular.json @@ -17,7 +17,6 @@ "build": { "builder": "@angular-devkit/build-angular:browser", "options": { - "aot": true, "outputPath": "backend/public", "index": "src/index.html", "main": "src/main.ts", @@ -31,9 +30,20 @@ "src/backend" ], "styles": [ - "src/styles.scss" + "src/styles.scss", + "src/bootstrap.min.css" ], - "scripts": [] + "scripts": [], + "vendorChunk": true, + "extractLicenses": false, + "buildOptimizer": false, + "sourceMap": true, + "optimization": false, + "namedChunks": true, + "allowedCommonJsDependencies": [ + "rxjs", + "crypto-js" + ] }, "configurations": { "production": { @@ -46,7 +56,6 @@ "optimization": true, "outputHashing": "all", "namedChunks": false, - "aot": true, "extractLicenses": true, "vendorChunk": false, "buildOptimizer": true, @@ -60,7 +69,8 @@ "es": { "localize": ["es"] } - } + }, + "defaultConfiguration": "" }, "serve": { "builder": "@angular-devkit/build-angular:dev-server", @@ -109,7 +119,8 @@ "src/backend" ], "styles": [ - "src/styles.scss" + "src/styles.scss", + "src/bootstrap.min.css" ], "scripts": [] }, @@ -142,7 +153,8 @@ "tsConfig": "src/tsconfig.spec.json", "scripts": [], "styles": [ - "src/styles.scss" + "src/styles.scss", + "src/bootstrap.min.css" ], "assets": [ "src/assets", @@ -152,16 +164,6 @@ "src/backend" ] } - }, - "lint": { - "builder": "@angular-devkit/build-angular:tslint", - "options": { - "tsConfig": [ - "src/tsconfig.app.json", - "src/tsconfig.spec.json" - ], - "exclude": [] - } } } }, @@ -176,20 +178,10 @@ "protractorConfig": "./protractor.conf.js", "devServerTarget": "youtube-dl-material:serve" } - }, - "lint": { - "builder": "@angular-devkit/build-angular:tslint", - "options": { - "tsConfig": [ - "e2e/tsconfig.e2e.json" - ], - "exclude": [] - } } } } }, - "defaultProject": "youtube-dl-material", "schematics": { "@schematics/angular:component": { "prefix": "app", @@ -198,5 +190,8 @@ "@schematics/angular:directive": { "prefix": "app" } + }, + "cli": { + "analytics": false } } \ No newline at end of file diff --git a/app.json b/app.json index 74ee445..30a7d86 100644 --- a/app.json +++ b/app.json @@ -2,6 +2,7 @@ "name": "YoutubeDL-Material", "description": "An open-source and self-hosted YouTube downloader based on Google's Material Design specifications.", "repository": "https://github.com/Tzahi12345/YoutubeDL-Material", + "stack": "container", "logo": "https://i.imgur.com/GPzvPiU.png", "keywords": ["youtube-dl", "youtubedl-material", "nodejs"] } \ No newline at end of file diff --git a/armhf.Dockerfile b/armhf.Dockerfile deleted file mode 100644 index 7f192be..0000000 --- a/armhf.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM alpine:3.12 as frontend - -RUN apk add --no-cache \ - npm \ - curl - -RUN npm install -g @angular/cli - -WORKDIR /build - -RUN curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . - -COPY [ "package.json", "package-lock.json", "/build/" ] -RUN npm install - -COPY [ "angular.json", "tsconfig.json", "/build/" ] -COPY [ "src/", "/build/src/" ] -RUN ng build --prod - -#--------------# - -FROM arm32v7/alpine:3.12 - -COPY --from=frontend /build/qemu-arm-static /usr/bin - -ENV UID=1000 \ - GID=1000 \ - USER=youtube - -RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID - -RUN apk add --no-cache \ - ffmpeg \ - npm \ - python2 \ - su-exec \ - && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \ - atomicparsley - -WORKDIR /app -COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ] -RUN npm install && chown -R $UID:$GID ./ - -COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] -COPY --chown=$UID:$GID [ "/backend/", "/app/" ] - -EXPOSE 17442 -ENTRYPOINT [ "/app/entrypoint.sh" ] -CMD [ "node", "app.js" ] diff --git a/backend/.eslintrc.json b/backend/.eslintrc.json new file mode 100644 index 0000000..172a2e4 --- /dev/null +++ b/backend/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "node": true, + "es2021": true + }, + "extends": [ + "eslint:recommended" + ], + "parser": "esprima", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [], + "rules": { + }, + "root": true +} diff --git a/backend/app.js b/backend/app.js index b9f3fab..43ab6bf 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1,45 +1,44 @@ const { uuid } = require('uuidv4'); -var fs = require('fs-extra'); -var { promisify } = require('util'); -var auth_api = require('./authentication/auth'); -var winston = require('winston'); -var path = require('path'); -var youtubedl = require('youtube-dl'); -var ffmpeg = require('fluent-ffmpeg'); -var compression = require('compression'); -var glob = require("glob") -var multer = require('multer'); -var express = require("express"); -var bodyParser = require("body-parser"); -var archiver = require('archiver'); -var unzipper = require('unzipper'); -var db_api = require('./db'); -var utils = require('./utils') -var mergeFiles = require('merge-files'); +const fs = require('fs-extra'); +const { promisify } = require('util'); +const auth_api = require('./authentication/auth'); +const winston = require('winston'); +const path = require('path'); +const compression = require('compression'); +const multer = require('multer'); +const express = require("express"); +const bodyParser = require("body-parser"); +const archiver = require('archiver'); +const unzipper = require('unzipper'); +const db_api = require('./db'); +const utils = require('./utils') const low = require('lowdb') -var ProgressBar = require('progress'); -const NodeID3 = require('node-id3') -const downloader = require('youtube-dl/lib/downloader') const fetch = require('node-fetch'); -var URL = require('url').URL; -const shortid = require('shortid') -const url_api = require('url'); -var config_api = require('./config.js'); -var subscriptions_api = require('./subscriptions') -var categories_api = require('./categories'); -var twitch_api = require('./twitch'); +const URL = require('url').URL; const CONSTS = require('./consts') -const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); -var ps = require('ps-node'); +const ps = require('ps-node'); +const Feed = require('feed').Feed; -const is_windows = process.platform === 'win32'; +// needed if bin/details somehow gets deleted +if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) + +const youtubedl = require('youtube-dl'); + +const logger = require('./logger'); +const config_api = require('./config.js'); +const downloader_api = require('./downloader'); +const tasks_api = require('./tasks'); +const subscriptions_api = require('./subscriptions'); +const categories_api = require('./categories'); +const twitch_api = require('./twitch'); +const youtubedl_api = require('./youtube-dl'); +const archive_api = require('./archive'); var app = express(); // database setup const FileSync = require('lowdb/adapters/FileSync'); -const config = require('./config.js'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -59,30 +58,9 @@ const admin_token = '4241b401-7236-493e-92b5-b72696b9d853'; // logging setup -// console format -const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => { - return `${timestamp} ${level.toUpperCase()}: ${message}`; -}); -const logger = winston.createLogger({ - level: 'info', - format: winston.format.combine(winston.format.timestamp(), defaultFormat), - defaultMeta: {}, - transports: [ - // - // - Write to all logs with level `info` and below to `combined.log` - // - Write all logs error (and below) to `error.log`. - // - new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), - new winston.transports.File({ filename: 'appdata/logs/combined.log' }), - new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'}) - ] -}); - -config_api.initialize(logger); -auth_api.initialize(db, users_db, logger); -db_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); -categories_api.initialize(db, users_db, logger, db_api); +config_api.initialize(); +db_api.initialize(db, users_db); +auth_api.initialize(db_api); // Set some defaults db.defaults( @@ -92,7 +70,9 @@ db.defaults( configWriteFlag: false, downloads: {}, subscriptions: [], - files_to_db_migration_complete: false + files_to_db_migration_complete: false, + tasks_manager_role_migration_complete: false, + archives_migration_complete: false }).write(); users_db.defaults( @@ -120,37 +100,32 @@ users_db.defaults( ).write(); // config values -var frontendUrl = null; -var backendUrl = null; -var backendPort = null; -var basePath = null; -var audioFolderPath = null; -var videoFolderPath = null; -var downloadOnlyMode = null; -var useDefaultDownloadingAgent = null; -var customDownloadingAgent = null; -var allowSubscriptions = null; -var subscriptionsCheckInterval = null; -var archivePath = path.join(__dirname, 'appdata', 'archives'); +let url = null; +let backendPort = null; +let useDefaultDownloadingAgent = null; +let customDownloadingAgent = null; +let allowSubscriptions = null; // other needed values -var url_domain = null; -var updaterStatus = null; +let url_domain = null; +let updaterStatus = null; -var timestamp_server_start = Date.now(); +const concurrentStreams = {}; if (debugMode) logger.info('YTDL-Material in debug mode!'); // check if just updated -const just_restarted = fs.existsSync('restart.json'); -if (just_restarted) { +const just_updated = fs.existsSync('restart_update.json'); +if (just_updated) { updaterStatus = { updating: false, details: 'Update complete! You are now on ' + CONSTS['CURRENT_VERSION'] } - fs.unlinkSync('restart.json'); + fs.unlinkSync('restart_update.json'); } +if (fs.existsSync('restart_general.json')) fs.unlinkSync('restart_general.json'); + // updates & starts youtubedl (commented out b/c of repo takedown) // startYoutubeDL(); @@ -166,97 +141,78 @@ var validDownloadingAgents = [ const subscription_timeouts = {}; +let version_info = null; +if (fs.existsSync('version.json')) { + version_info = fs.readJSONSync('version.json'); + logger.verbose(`Version info: ${JSON.stringify(version_info, null, 2)}`); +} else { + version_info = {'type': 'N/A', 'tag': 'N/A', 'commit': 'N/A', 'date': 'N/A'}; +} + // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); -let writeConfigMode = process.env.write_ytdl_config; // checks if config exists, if not, a config is auto generated config_api.configExistsCheck(); -if (writeConfigMode) { - setAndLoadConfig(); -} else { - loadConfig(); -} - -var downloads = {}; +setAndLoadConfig(); app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); // use passport app.use(auth_api.passport.initialize()); +app.use(auth_api.passport.session()); // actual functions -/** - * setTimeout, but its a promise. - * @param {number} ms - */ -async function wait(ms) { - await new Promise(resolve => { - setTimeout(resolve, ms); - }); -} - async function checkMigrations() { - // 3.5->3.6 migration - const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); - - if (!files_to_db_migration_complete) { - logger.info('Beginning migration: 3.5->3.6+') - const success = await runFilesToDBMigration() - if (success) { logger.info('3.5->3.6+ migration complete!'); } - else { logger.error('Migration failed: 3.5->3.6+'); } - } - // 4.1->4.2 migration const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value(); if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); - success = success && await addMetadataPropertyToDB('view_count'); - success = success && await addMetadataPropertyToDB('description'); - success = success && await addMetadataPropertyToDB('height'); - success = success && await addMetadataPropertyToDB('abr'); + success = success && await db_api.addMetadataPropertyToDB('view_count'); + success = success && await db_api.addMetadataPropertyToDB('description'); + success = success && await db_api.addMetadataPropertyToDB('height'); + success = success && await db_api.addMetadataPropertyToDB('abr'); + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } - return true; -} - -async function runFilesToDBMigration() { - try { - let mp3s = await getMp3s(); - let mp4s = await getMp4s(); - - for (let i = 0; i < mp3s.length; i++) { - let file_obj = mp3s[i]; - const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); - } - } - - for (let i = 0; i < mp4s.length; i++) { - let file_obj = mp4s[i]; - const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp4', 'video'); - } - } - + const new_db_system_migration_complete = db.get('new_db_system_migration_complete').value(); + if (!new_db_system_migration_complete) { + logger.info('Beginning migration: 4.2->4.3+') + let success = await db_api.importJSONToDB(db.value(), users_db.value()); + await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first // sets migration to complete - db.set('files_to_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; + db.set('new_db_system_migration_complete', true).write(); + if (success) { logger.info('4.2->4.3+ migration complete!'); } + else { logger.error('Migration failed: 4.2->4.3+'); } } + + const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value(); + if (!tasks_manager_role_migration_complete) { + logger.info('Checking if tasks manager role permissions exist for admin user...'); + const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes'); + if (success) logger.info('Task manager permissions check complete!'); + else logger.error('Failed to auto add tasks manager permissions to admin role!'); + db.set('tasks_manager_role_migration_complete', true).write(); + } + + const archives_migration_complete = db.get('archives_migration_complete').value(); + if (!archives_migration_complete) { + logger.info('Checking if archives have been migrated...'); + const imported_archives = await archive_api.importArchives(); + if (imported_archives) logger.info('Archives migration complete!'); + else logger.error('Failed to migrate archives!'); + db.set('archives_migration_complete', true).write(); + } + + return true; } async function simplifyDBFileStructure() { @@ -296,28 +252,6 @@ async function simplifyDBFileStructure() { return true; } -async function addMetadataPropertyToDB(property_key) { - try { - const dirs_to_check = db_api.getFileDirectoriesAndDBs(); - for (const dir_to_check of dirs_to_check) { - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); - for (const file of files) { - if (file[property_key]) { - dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); - } - } - } - - // sets migration to complete - db.set('simplified_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; - } -} - async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku @@ -332,20 +266,6 @@ async function startServer() { }); } -async function restartServer() { - const restartProcess = () => { - spawn('node', ['app.js'], { - detached: true, - stdio: 'inherit' - }).unref() - process.exit() - } - logger.info('Update complete! Restarting server...'); - - // the following line restarts the server through nodemon - fs.writeFileSync('restart.json', 'internal use only'); -} - async function updateServer(tag) { // no tag provided means update to the latest version if (!tag) { @@ -386,8 +306,9 @@ async function updateServer(tag) { updating: true, 'details': 'Update complete! Restarting server...' } - restartServer(); + utils.restartServer(true); }, err => { + logger.error(err); updaterStatus = { updating: false, error: true, @@ -418,12 +339,10 @@ async function downloadReleaseFiles(tag) { fs.createReadStream(path.join(__dirname, `youtubedl-material-release-${tag}.zip`)).pipe(unzipper.Parse()) .on('entry', function (entry) { var fileName = entry.path; - var type = entry.type; // 'Directory' or 'File' - var size = entry.size; var is_dir = fileName.substring(fileName.length-1, fileName.length) === '/' if (!is_dir && fileName.includes('youtubedl-material/public/')) { // get public folder files - var actualFileName = fileName.replace('youtubedl-material/public/', ''); + const actualFileName = fileName.replace('youtubedl-material/public/', ''); if (actualFileName.length !== 0 && actualFileName.substring(actualFileName.length-1, actualFileName.length) !== '/') { fs.ensureDirSync(path.join(__dirname, 'public', path.dirname(actualFileName))); entry.pipe(fs.createWriteStream(path.join(__dirname, 'public', actualFileName))); @@ -432,7 +351,7 @@ async function downloadReleaseFiles(tag) { } } else if (!is_dir && !replace_ignore_list.includes(fileName)) { // get package.json - var actualFileName = fileName.replace('youtubedl-material/', ''); + const actualFileName = fileName.replace('youtubedl-material/', ''); logger.verbose('Downloading file ' + actualFileName); entry.pipe(fs.createWriteStream(path.join(__dirname, actualFileName))); } else { @@ -445,34 +364,6 @@ async function downloadReleaseFiles(tag) { }); } -// helper function to download file using fetch -async function fetchFile(url, path, file_label) { - var len = null; - const res = await fetch(url); - - len = parseInt(res.headers.get("Content-Length"), 10); - - var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, { - complete: '=', - incomplete: ' ', - width: 20, - total: len - }); - const fileStream = fs.createWriteStream(path); - await new Promise((resolve, reject) => { - res.body.pipe(fileStream); - res.body.on("error", (err) => { - reject(err); - }); - res.body.on('data', function (chunk) { - bar.tick(chunk.length); - }); - fileStream.on("finish", function() { - resolve(); - }); - }); - } - async function downloadReleaseZip(tag) { return new Promise(async resolve => { // get name of zip file, which depends on the version @@ -483,7 +374,7 @@ async function downloadReleaseZip(tag) { let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`); // download zip from release - await fetchFile(latest_zip_link, output_path, 'update ' + tag); + await utils.fetchFile(latest_zip_link, output_path, 'update ' + tag); resolve(true); }); @@ -531,7 +422,7 @@ async function backupServerLite() { }); // wait a tiny bit for the zip to reload in fs - await wait(100); + await utils.wait(100); return true; } @@ -561,9 +452,10 @@ async function getLatestVersion() { async function killAllDownloads() { const lookupAsync = promisify(ps.lookup); + let resultList = null; try { - await lookupAsync({ + resultList = await lookupAsync({ command: 'youtube-dl' }); } catch (err) { @@ -599,7 +491,7 @@ async function killAllDownloads() { async function setPortItemFromENV() { config_api.setConfigItem('ytdl_port', backendPort.toString()); - await wait(100); + await utils.wait(100); return true; } @@ -609,11 +501,12 @@ async function setAndLoadConfig() { } async function setConfigFromEnv() { - let config_items = getEnvConfigItems(); - let success = config_api.setConfigItems(config_items); + const config_items = getEnvConfigItems(); + if (!config_items || config_items.length === 0) return true; + const success = config_api.setConfigItems(config_items); if (success) { logger.info('Config items set using ENV variables.'); - await wait(100); + await utils.wait(100); return true; } else { logger.error('ERROR: Failed to set config items using ENV variables.'); @@ -624,8 +517,11 @@ async function setConfigFromEnv() { async function loadConfig() { loadConfigValues(); - // creates archive path if missing - await fs.ensureDir(archivePath); + // connect to DB + if (!config_api.getConfigItem('ytdl_use_local_db')) + await db_api.connectToDB(); + db_api.database_initialized = true; + db_api.database_initialized_bs.next(true); // check migrations await checkMigrations(); @@ -636,20 +532,18 @@ async function loadConfig() { // get subscriptions if (allowSubscriptions) { // set downloading to false - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds - watchSubscriptions(); - setInterval(() => { + const watchSubscriptionsInterval = function() { watchSubscriptions(); - }, subscriptionsCheckInterval * 1000); + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); + setTimeout(watchSubscriptionsInterval, subscriptionsCheckInterval*1000); + } + + watchSubscriptionsInterval(); } - db_api.importUnregisteredFiles(); - - // load in previous downloads - downloads = db.get('downloads').value(); - // start the server here startServer(); @@ -659,13 +553,9 @@ async function loadConfig() { function loadConfigValues() { url = !debugMode ? config_api.getConfigItem('ytdl_url') : 'http://localhost:4200'; backendPort = config_api.getConfigItem('ytdl_port'); - audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); - videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode'); useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent'); customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent'); allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions'); - subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { logger.info(`Using non-default downloading agent \'${customDownloadingAgent}\'`) @@ -678,29 +568,27 @@ function loadConfigValues() { url_domain = new URL(url); let logger_level = config_api.getConfigItem('ytdl_logger_level'); - const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug']; - if (!possible_levels.includes(logger_level)) { - logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`) - logger_level = 'info'; - } - logger.level = logger_level; - winston.loggers.get('console').level = logger_level; - logger.transports[2].level = logger_level; + utils.updateLoggerLevel(logger_level); } function calculateSubcriptionRetrievalDelay(subscriptions_amount) { // frequency is once every 5 mins by default + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); let interval_in_ms = subscriptionsCheckInterval * 1000; const subinterval_in_ms = interval_in_ms/subscriptions_amount; return subinterval_in_ms; } async function watchSubscriptions() { - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; - const valid_subscriptions = subscriptions.filter(sub => !sub.paused); + // auto pause deprecated streamingOnly mode + const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly); + subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true}); + + const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly); let subscriptions_amount = valid_subscriptions.length; let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount); @@ -734,6 +622,7 @@ async function watchSubscriptions() { }, current_delay); subscription_timeouts[sub.id] = true; current_delay += delay_interval; + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; } } @@ -763,806 +652,17 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -function getThumbnailMp3(name) -{ - var obj = utils.getJSONMp3(name, audioFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getThumbnailMp4(name) -{ - var obj = utils.getJSONMp4(name, videoFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getFileSizeMp3(name) -{ - var jsonPath = audioFolderPath+name+".mp3.info.json"; - - if (fs.existsSync(jsonPath)) - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - else - var obj = 0; - - return obj.filesize; -} - -function getFileSizeMp4(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - var filesize = 0; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - for (i = 0; i < obj.formats.length; i++) - { - if (obj.formats[i].format_id == format) - { - filesize = obj.formats[i].filesize; - } - } - } - - return filesize; -} - -function getAmountDownloadedMp3(name) -{ - var partPath = audioFolderPath+name+".mp3.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - - - -function getAmountDownloadedMp4(name) -{ - var format = getVideoFormatID(name); - var partPath = videoFolderPath+name+".f"+format+".mp4.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - -function getVideoFormatID(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - return format; - } -} - -async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { - let zipFolderPath = null; - - if (!fullPathProvided) { - zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath); - if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); - } else { - zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - - let ext = (type === 'audio') ? '.mp3' : '.mp4'; - - let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip')); - - var archive = archiver('zip', { - gzip: true, - zlib: { level: 9 } // Sets the compression level. - }); - - archive.on('error', function(err) { - logger.error(err); - throw err; - }); - - // pipe archive data to the output file - archive.pipe(output); - - for (let i = 0; i < fileNames.length; i++) { - let fileName = fileNames[i]; - let fileNamePathRemoved = path.parse(fileName).base; - let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName; - archive.file(file_path, {name: fileNamePathRemoved + ext}) - } - - await archive.finalize(); - - // wait a tiny bit for the zip to reload in fs - await wait(100); - return path.join(zipFolderPath,outputName + '.zip'); -} - -async function deleteAudioFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : audioFolderPath; - - var jsonPath = path.join(filePath,name+'.mp3.info.json'); - var altJSONPath = path.join(filePath,name+'.info.json'); - var audioFilePath = path.join(filePath,name+'.mp3'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - altJSONPath = path.join(__dirname, altJSONPath); - audioFilePath = path.join(__dirname, audioFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - let audioFileExists = await fs.pathExists(audioFilePath); - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_audio.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp3(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('audio', line); - } else { - logger.info('Could not find archive file for audio files. Creating...'); - await fs.close(await fs.open(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (audioFileExists) { - await fs.unlink(audioFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(audioFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -async function deleteVideoFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : videoFolderPath; - var jsonPath = path.join(filePath,name+'.info.json'); - - var altJSONPath = path.join(filePath,name+'.mp4.info.json'); - var videoFilePath = path.join(filePath,name+'.mp4'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - videoFilePath = path.join(__dirname, videoFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let videoFileExists = await fs.pathExists(videoFilePath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp4(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('video', line); - } else { - logger.info('Could not find archive file for videos. Creating...'); - fs.closeSync(fs.openSync(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (videoFileExists) { - await fs.unlink(videoFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(videoFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -/** - * @param {'audio' | 'video'} type - * @param {string[]} fileNames - */ -async function getAudioOrVideoInfos(type, fileNames) { - let result = await Promise.all(fileNames.map(async fileName => { - let fileLocation = videoFolderPath+fileName; - if (type === 'audio') { - fileLocation += '.mp3.info.json'; - } else if (type === 'video') { - fileLocation += '.info.json'; - } - - if (await fs.pathExists(fileLocation)) { - let data = await fs.readFile(fileLocation); - try { - return JSON.parse(data); - } catch (e) { - let suffix; - if (type === 'audio') { - suffix += '.mp3'; - } else if (type === 'video') { - suffix += '.mp4'; - } - - logger.error(`Could not find info for file ${fileName}${suffix}`); - } - } - return null; - })); - - return result.filter(data => data != null); -} - -// downloads - -async function downloadFileByURL_exec(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - - // audio / video specific vars - var is_audio = type === 'audio'; - var ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; - let category = null; - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'exec'; - let downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now(), - filesize: null - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - let download_checker = null; - - // get video info prior to download - let info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info && url.includes('youtu')) { - resolve(false); - return; - } else if (info) { - // check if it fits into a category. If so, then get info again using new downloadConfig - category = await categories_api.categorize(info); - - // set custom output if the category has one and re-retrieve info so the download manager has the right file name - if (category && category['custom_output']) { - options.customOutput = category['custom_output']; - options.noRelativePath = true; - downloadConfig = await generateArgs(url, type, options); - info = await getVideoInfoByURL(url, downloadConfig, download); - } - - // store info in download for future use - download['_filename'] = info['_filename']; - download['filesize'] = utils.getExpectedFileSize(info); - download_checker = setInterval(() => checkDownloadPercent(download), 1000); - } - - // download file - youtubedl.exec(url, downloadConfig, {}, function(err, output) { - if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) - - download['downloading'] = false; - download['timestamp_end'] = Date.now(); - var file_uid = null; - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); - if (err) { - logger.error(err.stderr); - - download['error'] = err.stderr; - updateDownloads(); - resolve(false); - return; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { - download['error'] = 'No output. Check if video already exists in your archive.'; - logger.warn(`No output received for video download, check if it exists in your archive.`) - updateDownloads(); - - resolve(false); - return; - } - var file_names = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - - if (!output_json) { - continue; - } - - // get filepath with no extension - const filepath_no_extension = utils.removeFileExtension(output_json['_filename']); - - var full_file_path = filepath_no_extension + ext; - var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); - - if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 - && config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) { - let vodId = url.split('twitch.tv/videos/')[1]; - vodId = vodId.split('?')[0]; - twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user); - } - - // renames file if necessary due to bug - if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { - try { - fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); - logger.info('Renamed ' + file_name + '.webm to ' + file_name); - } catch(e) { - } - } - - if (type === 'audio') { - let tags = { - title: output_json['title'], - artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] - } - let success = NodeID3.write(tags, output_json['_filename']); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); - } - - const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); - const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; - - // registers file in DB - file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category); - - if (file_name) file_names.push(file_name); - } - - let is_playlist = file_names.length > 1; - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - download['complete'] = true; - download['fileNames'] = is_playlist ? file_names : [full_file_path] - updateDownloads(); - - var videopathEncoded = encodeURIComponent(file_names[0]); - - resolve({ - [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: is_playlist ? file_names : null, - uid: file_uid - }); - } - }); - }); -} - -async function downloadFileByURL_normal(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - var file_uid = null; - const is_audio = type === 'audio'; - const ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; } - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'normal'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - const video = youtubedl(url, - // Optional arguments passed to youtube-dl. - downloadConfig, - // Additional options can be given for calling `child_process.execFile()`. - { cwd: __dirname }); - - let video_info = null; - let file_size = 0; - - // Will be called when the download starts. - video.on('info', function(info) { - video_info = info; - file_size = video_info.size; - const json_path = utils.removeFileExtension(video_info._filename) + '.info.json'; - fs.ensureFileSync(json_path); - fs.writeJSONSync(json_path, video_info); - video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) - }); - // Will be called if download was already completed and there is nothing more to download. - video.on('complete', function complete(info) { - 'use strict' - logger.info('file ' + info._filename + ' already downloaded.') - }) - - let download_pos = 0; - video.on('data', function data(chunk) { - download_pos += chunk.length - // `size` should not be 0 here. - if (file_size) { - let percent = (download_pos / file_size * 100).toFixed(2) - download['percent_complete'] = percent; - } - }); - - video.on('end', async function() { - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - download['timestamp_end'] = Date.now(); - download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext]; - download['complete'] = true; - updateDownloads(); - - // audio-only cleanup - if (is_audio) { - // filename fix - video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3'; - - // ID3 tagging - let tags = { - title: video_info['title'], - artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] - } - let success = NodeID3.write(tags, video_info._filename); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); - - const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm'; - const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4'; - // check if audio file is webm - if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']); - else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']); - } - - // registers file in DB - const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name)); - - resolve({ - [is_audio ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready - uid: file_uid - }); - }); - - video.on('error', function error(err) { - logger.error(err); - - download[error] = err; - updateDownloads(); - - resolve(false); - }); - }); - -} - -async function generateArgs(url, type, options) { - var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; - var globalArgs = config_api.getConfigItem('ytdl_custom_args'); - let useCookies = config_api.getConfigItem('ytdl_use_cookies'); - var is_audio = type === 'audio'; - - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath; - - var customArgs = options.customArgs; - var customOutput = options.customOutput; - var customQualityConfiguration = options.customQualityConfiguration; - - // video-specific args - var selectedHeight = options.selectedHeight; - - // audio-specific args - var maxBitrate = options.maxBitrate; - - var youtubeUsername = options.youtubeUsername; - var youtubePassword = options.youtubePassword; - - let downloadConfig = null; - let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4']; - const is_youtube = url.includes('youtu'); - if (!is_audio && !is_youtube) { - // tiktok videos fail when using the default format - qualityPath = null; - } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) { - qualityPath = ['-f', 'bestvideo+bestaudio'] - } - - if (customArgs) { - downloadConfig = customArgs.split(',,'); - } else { - if (customQualityConfiguration) { - qualityPath = ['-f', customQualityConfiguration]; - } else if (selectedHeight && selectedHeight !== '' && !is_audio) { - qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; - } else if (maxBitrate && is_audio) { - qualityPath = ['--audio-quality', maxBitrate] - } - - if (customOutput) { - customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); - downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; - } else { - downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; - } - - if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath); - - if (is_audio && !options.skip_audio_args) { - downloadConfig.push('-x'); - downloadConfig.push('--audio-format', 'mp3'); - } - - if (youtubeUsername && youtubePassword) { - downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); - } - - if (useCookies) { - if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) { - downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); - } else { - logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); - } - } - - if (!useDefaultDownloadingAgent && customDownloadingAgent) { - downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath; - const archive_path = path.join(archive_folder, `archive_${type}.txt`); - - await fs.ensureDir(archive_folder); - - // create archive file if it doesn't exist - if (!(await fs.pathExists(archive_path))) { - await fs.close(await fs.open(archive_path, 'w')); - } - - let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); - // create blacklist file if it doesn't exist - if (!(await fs.pathExists(blacklist_path))) { - await fs.close(await fs.open(blacklist_path, 'w')); - } - - let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); - await fs.ensureFile(merged_path); - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - let status = await mergeFiles(inputPathList, merged_path); - - options.merged_string = await fs.readFile(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - downloadConfig.push('--write-thumbnail'); - } - - if (globalArgs && globalArgs !== '') { - // adds global args - if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) { - // if global args has an output, replce the original output with that of global args - const original_output_index = downloadConfig.indexOf('-o'); - downloadConfig.splice(original_output_index, 2); - } - downloadConfig = downloadConfig.concat(globalArgs.split(',,')); - } - - } - - // filter out incompatible args - downloadConfig = filterArgs(downloadConfig, is_audio); - - logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); - return downloadConfig; -} - -async function getVideoInfoByURL(url, args = [], download = null) { - return new Promise(resolve => { - // remove bad args - const new_args = [...args]; - - const archiveArgIndex = new_args.indexOf('--download-archive'); - if (archiveArgIndex !== -1) { - new_args.splice(archiveArgIndex, 2); - } - - // actually get info - youtubedl.getInfo(url, new_args, (err, output) => { - if (output) { - resolve(output); - } else { - logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`); - if (download) { - download['error'] = `Failed pre-check for video info: ${err}`; - updateDownloads(); - } - resolve(null); - } - }); - }); -} - -function filterArgs(args, isAudio) { - const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs']; - const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail']; - const args_to_remove = isAudio ? video_only_args : audio_only_args; - return args.filter(x => !args_to_remove.includes(x)); -} - // currently only works for single urls -async function getUrlInfos(urls) { +async function getUrlInfos(url) { let startDate = Date.now(); let result = []; return new Promise(resolve => { - youtubedl.exec(urls.join(' '), ['--dump-json'], {}, (err, output) => { + youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { let new_date = Date.now(); let difference = (new_date - startDate)/1000; logger.debug(`URL info retrieval delay: ${difference} seconds.`); if (err) { - logger.error('Error during parsing:' + err); + logger.error(`Error during retrieving formats for ${url}: ${err}`); resolve(null); } let try_putput = null; @@ -1570,213 +670,20 @@ async function getUrlInfos(urls) { try_putput = JSON.parse(output); result = try_putput; } catch(e) { - // probably multiple urls - logger.error('failed to parse for urls starting with ' + urls[0]); - // logger.info(output); + logger.error(`Failed to retrieve available formats for url: ${url}`); } resolve(result); }); }); } -async function convertFileToMp3(input_file, output_file) { - logger.verbose(`Converting ${input_file} to ${output_file}...`); - return new Promise(resolve => { - ffmpeg(input_file).noVideo().toFormat('mp3') - .on('end', () => { - logger.verbose(`Conversion for '${output_file}' complete.`); - fs.unlinkSync(input_file) - resolve(true); - }) - .on('error', (err) => { - logger.error('Failed to convert audio file to the correct format.'); - logger.error(err); - resolve(false); - }).save(output_file); - }); -} - -async function writeToBlacklist(type, line) { - let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); - // adds newline to the beginning of the line - line = '\n' + line; - await fs.appendFile(blacklistPath, line); -} - -// download management functions - -function updateDownloads() { - db.assign({downloads: downloads}).write(); -} - -function checkDownloadPercent(download) { - /* - This is more of an art than a science, we're just selecting files that start with the file name, - thus capturing the parts being downloaded in files named like so: '