diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..c5624c4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,20 @@ +{ + "env": { + "browser": true, + "es2021": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [ + "@typescript-eslint" + ], + "rules": { + } +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..94f04ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** +- YoutubeDL-Material version +- Docker tag: (optional) + +Ideally you'd copy the info as presented on the "About" dialogue +in YoutubeDL-Material. +(for that, click on the three dots on the top right and then +check "installation details". On later versions of YoutubeDL- +Material you will find pretty much all the crucial information +here that we need in most cases!) + +**Additional context** +Add any other context about the problem here. For example, a YouTube link. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d233fde --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]" +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3b911e..50f4eb5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,15 +15,31 @@ jobs: - name: checkout code uses: actions/checkout@v2 - name: setup node - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 + with: + node-version: '12' + cache: 'npm' - name: install dependencies run: | npm install cd backend npm install sudo npm install -g @angular/cli + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' - name: build - run: ng build --prod + run: npm run build - name: prepare artifact upload shell: pwsh run: | @@ -35,7 +51,7 @@ jobs: Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material - New-Item -Path ./build/youtubedl-material -Name users + New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material - name: upload build artifact diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 96ec2c8..3063a97 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -13,6 +13,19 @@ jobs: steps: - name: checkout code uses: actions/checkout@v2 + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' - name: setup platform emulator uses: docker/setup-qemu-action@v1 - name: setup multi-arch docker build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c74f29b..3fd4ab2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,19 @@ jobs: steps: - name: checkout code uses: actions/checkout@v2 + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' - name: setup platform emulator uses: docker/setup-qemu-action@v1 - name: setup multi-arch docker build diff --git a/.gitignore b/.gitignore index 9d2ac87..44fa9c2 100644 --- a/.gitignore +++ b/.gitignore @@ -65,4 +65,5 @@ backend/appdata/logs/error.log backend/appdata/users.json backend/users/* backend/appdata/cookies.txt -backend/public \ No newline at end of file +backend/public +src/assets/i18n/*.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..6db05be --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,25 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "start", + "problemMatcher": [], + "label": "Dev: start frontend", + "detail": "ng serve" + }, + { + "label": "Dev: start backend", + "type": "shell", + "command": "set YTDL_MODE=debug && node app.js", + "options": { + "cwd": "./backend" + }, + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [] + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 75b22d3..ead755e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.12 as frontend +FROM alpine:latest as frontend RUN apk add --no-cache \ npm @@ -11,28 +11,33 @@ RUN npm install COPY [ "angular.json", "tsconfig.json", "/build/" ] COPY [ "src/", "/build/src/" ] -RUN ng build --prod +RUN npm run build #--------------# -FROM alpine:3.12 +FROM alpine:latest ENV UID=1000 \ GID=1000 \ USER=youtube +ENV NO_UPDATE_NOTIFIER=true + RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID RUN apk add --no-cache \ ffmpeg \ npm \ python2 \ + python3 \ su-exec \ && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \ atomicparsley WORKDIR /app COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ] +ENV PM2_HOME=/app/pm2 +RUN npm install pm2 -g RUN npm install && chown -R $UID:$GID ./ COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] @@ -40,4 +45,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ] EXPOSE 17442 ENTRYPOINT [ "/app/entrypoint.sh" ] -CMD [ "node", "app.js" ] +CMD [ "pm2-runtime", "pm2.config.js" ] diff --git a/Public API v1.yaml b/Public API v1.yaml index b57c386..ad43d48 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -3,35 +3,14 @@ info: title: YoutubeDL-Material API Docs - Official version: '1.0' description: "Welcome to the official docs for YoutubeDL-Material.\n\n\nYou can check out all the available endpoints. Remember to authenticate with your API key using the \"apiKey\" query parameter with your requests.\n\n\nTo do so, simply add this to the end of your API call:\n\n\n`?apiKey=API_KEY`\n\n\nReplce API_KEY with the API key in the settings menu. If one doesn't exist, click generate to create one.\n\n### Multi-user mode\n\nWhen using multi-user mode, you will need to supply a JWT token to authenticate requests through that user. This lets the server know which user to run the task for, like downloading a video for a specific user. \n\nTo do this, you must use the `/api/auth/login` endpoint to login using a user's username and password. This will result in an object containing a `token`. Supply this along with your API key like so:\n\n`?apiKey=API_KEY&jwt=JWT_TOKEN`\n\nNotice the `&` between the `API_KEY` and `jwt`." + contact: + name: Isaac Abadi + url: https://github.com/Tzahi12345/YoutubeDL-Material + email: IsaacMGrynsztein@gmail.com servers: - url: 'http://localhost:17442' paths: - /api/tomp3: - post: - tags: - - downloader - summary: Download audio file - description: |- - Downloads an audio file with the given URL. Will include global args if they exist. - - - HTTP requests will return once the audio file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status. - operationId: post-tomp3 - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200' - security: - - Auth query parameter: [] - /api/tomp4: + /api/downloadFile: post: tags: - downloader @@ -46,14 +25,37 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_1' + $ref: '#/components/schemas/DownloadRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_1' + $ref: '#/components/schemas/DownloadResponse' + '500': + description: Server download error + security: + - Auth query parameter: [] + /api/generateArgs: + post: + tags: + - downloader + summary: Download video file + description: Generates args, used for checking what args would run if you ran downloadFile + operationId: post-generateArgs + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GenerateArgsResponse' security: - Auth query parameter: [] /api/getMp3s: @@ -69,8 +71,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_2' - requestBody: {} + $ref: '#/components/schemas/GetMp3sResponse' security: - Auth query parameter: [] /api/getMp4s: @@ -86,7 +87,23 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_3' + $ref: '#/components/schemas/GetMp4sResponse' + security: + - Auth query parameter: [] + /api/getAllFiles: + post: + tags: + - files + summary: Get all files + description: Gets all files and playlists stored in the db + operationId: get-getAllFiles + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllFilesResponse' security: - Auth query parameter: [] /api/getFile: @@ -100,14 +117,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_2' + $ref: '#/components/schemas/GetFileRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_4' + $ref: '#/components/schemas/GetFileResponse' + '401': + description: User is not authorized to view the file. security: - Auth query parameter: [] /api/enableSharing: @@ -122,14 +141,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_3' + $ref: '#/components/schemas/SharingToggle' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/disableSharing: @@ -144,14 +163,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_4' + $ref: '#/components/schemas/SharingToggle' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/incrementViewCount: + post: + summary: Increments a file's view count + tags: + - files + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/IncrementViewCountRequest' + operationId: post-api-incrementViewCount + description: Increments a file's view count security: - Auth query parameter: [] /api/subscribe: @@ -165,14 +205,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_5' + $ref: '#/components/schemas/SubscribeRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_6' + $ref: '#/components/schemas/SubscribeResponse' security: - Auth query parameter: [] /api/unsubscribe: @@ -186,14 +226,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_6' + $ref: '#/components/schemas/UnsubscribeRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_7' + $ref: '#/components/schemas/UnsubscribeResponse' security: - Auth query parameter: [] /api/deleteSubscriptionFile: @@ -207,14 +247,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_7' + $ref: '#/components/schemas/DeleteSubscriptionFileRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_8' + $ref: '#/components/schemas/SuccessObject' '500': description: Internal Server Error security: @@ -230,14 +270,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_8' + $ref: '#/components/schemas/GetSubscriptionRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_9' + $ref: '#/components/schemas/GetSubscriptionResponse' security: - Auth query parameter: [] /api/downloadVideosForSubscription: @@ -251,14 +291,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_9' + $ref: '#/components/schemas/DownloadVideosForSubscriptionRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_10' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/getSubscriptions: @@ -278,7 +318,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_11' + $ref: '#/components/schemas/GetAllSubscriptionsResponse' security: - Auth query parameter: [] /api/createPlaylist: @@ -292,14 +332,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_10' + $ref: '#/components/schemas/CreatePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_12' + $ref: '#/components/schemas/CreatePlaylistResponse' security: - Auth query parameter: [] /api/getPlaylist: @@ -313,35 +353,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_11' + $ref: '#/components/schemas/GetPlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_13' + $ref: '#/components/schemas/GetPlaylistResponse' security: - Auth query parameter: [] /api/updatePlaylist: post: tags: - playlists - summary: Update playlist files - description: Updates the list of filenames in the playlist object + summary: Update playlist + description: Updates the playlist object operationId: post-api-updatePlaylist requestBody: content: application/json: schema: - $ref: '#/components/schemas/body_12' + $ref: '#/components/schemas/UpdatePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/deletePlaylist: @@ -355,14 +395,35 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_13' + $ref: '#/components/schemas/DeletePlaylistRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/addFileToPlaylist: + post: + tags: + - playlists + summary: Adds a file to a playlist + description: Adds a file to a playlist + operationId: post-api-addFileToPlaylist + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AddFileToPlaylistRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/deleteMp4: @@ -376,7 +437,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_14' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK @@ -387,7 +448,7 @@ paths: description: Whether the operation succeeded security: - Auth query parameter: [] - /api/downloadFile: + /api/downloadFileFromServer: post: tags: - files @@ -397,27 +458,25 @@ paths: content: application/json: schema: - type: object - properties: {} - application/xml: - schema: - $ref: '#/components/schemas/body_15' + $ref: '#/components/schemas/DownloadFileRequest' responses: '200': description: 'The file itself is in the response, as well as an options object.' + '401': + description: User is not authorized to view the file. security: - Auth query parameter: [] /api/deleteFile: post: tags: - files - summary: Delete downloaded file (unused) + summary: Delete downloaded file operationId: post-api-deleteFile requestBody: content: application/json: schema: - $ref: '#/components/schemas/body_16' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK @@ -434,10 +493,12 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_17' + $ref: '#/components/schemas/DownloadArchiveRequest' responses: '200': description: The archive text file is sent as a response + '404': + description: If the archive dir is not found, 404 is sent as a response security: - Auth query parameter: [] /api/updaterStatus: @@ -453,7 +514,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_14' + $ref: '#/components/schemas/UpdaterStatus' security: - Auth query parameter: [] /api/updateServer: @@ -467,14 +528,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_18' + $ref: '#/components/schemas/UpdateServerRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/isPinSet: @@ -493,48 +554,6 @@ paths: $ref: '#/components/schemas/inline_response_200_15' security: - Auth query parameter: [] - /api/checkPin: - post: - tags: - - security - summary: Check if pin is correct - description: Checks the pin against an inputted one. Will return true if they match - operationId: post-api-checkPin - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body_19' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200_5' - security: - - Auth query parameter: [] - /api/setPin: - post: - tags: - - security - summary: Set pin - operationId: post-api-setPin - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/body_20' - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_200_5' - security: - - Auth query parameter: [] - description: '' /api/generateNewAPIKey: post: tags: @@ -548,7 +567,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_16' + $ref: '#/components/schemas/GenerateNewApiKeyResponse' security: - Auth query parameter: [] /api/deleteMp3: @@ -562,7 +581,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_21' + $ref: '#/components/schemas/DeleteMp3Mp4Request' responses: '200': description: OK @@ -585,7 +604,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_17' + $ref: '#/components/schemas/ConfigResponse' security: - Auth query parameter: [] /api/setConfig: @@ -597,18 +616,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/body_22' + $ref: '#/components/schemas/SetConfigRequest' responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/inline_response_200_5' + $ref: '#/components/schemas/SuccessObject' + '404': + description: Tried to save invalid config file. security: - Auth query parameter: [] /api/downloads: - get: + post: summary: Get info for all downloads tags: - downloader @@ -618,34 +639,13 @@ paths: content: application/json: schema: - type: object - properties: - downloads: - type: object - properties: - uid: - type: string - downloading: - type: boolean - complete: - type: boolean - url: - type: string - type: - type: string - percent_complete: - type: string - is_playlist: - type: boolean - timestamp_start: - type: number - timestamp_end: - type: number - fileNames: - type: array - items: - type: string - operationId: get-api-downloads + $ref: '#/components/schemas/GetAllDownloadsResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllDownloadsRequest' + operationId: post-api-downloads description: Retrieves all downloads recorded by the server and their status. security: - Auth query parameter: [] @@ -659,46 +659,12 @@ paths: content: application/json: schema: - type: object - properties: - download: - type: object - properties: - uid: - type: string - downloading: - type: boolean - complete: - type: boolean - url: - type: string - type: - type: string - percent_complete: - type: string - is_playlist: - type: boolean - timestamp_start: - type: number - timestamp_end: - type: number - fileNames: - type: array - items: - type: string + $ref: '#/components/schemas/GetDownloadResponse' requestBody: content: application/json: schema: - type: object - properties: - session_id: - type: string - download_id: - type: string - required: - - session_id - - download_id + $ref: '#/components/schemas/GetDownloadRequest' description: '' description: "Gets a single download using its download_id and session_id. session_id is the device fingerprint. If none was provided at the time of download, then set session_id is 'undeclared'." security: @@ -715,35 +681,13 @@ paths: content: application/json: schema: - type: object - properties: - user: - $ref: '#/components/schemas/user' - token: - type: string - permissions: - type: array - items: - type: string - available_permissions: - type: array - items: - type: string + $ref: '#/components/schemas/LoginResponse' description: Use this method to log into a user using their username and password and receive a jwt auth token so you can send per-user requests. requestBody: content: application/json: schema: - type: object - properties: - userid: - type: string - description: 'This is the username, not the user uid' - password: - type: string - required: - - userid - - password + $ref: '#/components/schemas/LoginRequest' security: - Auth query parameter: [] tags: @@ -758,27 +702,13 @@ paths: content: application/json: schema: - type: object - properties: - user: - $ref: '#/components/schemas/user' + $ref: '#/components/schemas/RegisterResponse' description: Use this endpoint to register a user. It will only work if registration is enabled. requestBody: content: application/json: schema: - type: object - properties: - userid: - type: string - username: - type: string - password: - type: string - required: - - userid - - username - - password + $ref: '#/components/schemas/RegisterRequest' security: - Auth query parameter: [] tags: @@ -790,21 +720,15 @@ paths: responses: '200': description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' requestBody: content: application/json: schema: - type: object - properties: - change_object: - type: object - properties: - name: - type: string - role: - type: string - required: - - change_object + $ref: '#/components/schemas/UpdateUserRequest' description: Updates certain properties for a user. Only two are possible right now. security: - Auth query parameter: [] @@ -820,21 +744,13 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' description: Deletes a user by its uid. requestBody: content: application/json: schema: - type: object - properties: - uid: - type: string - required: - - uid + $ref: '#/components/schemas/DeleteUserRequest' security: - Auth query parameter: [] tags: @@ -849,25 +765,7 @@ paths: content: application/json: schema: - type: object - properties: - roles: - type: object - properties: - admin: - type: object - properties: - permissions: - type: array - items: - type: string - user: - type: object - properties: - permissions: - type: array - items: - type: string + $ref: '#/components/schemas/GetRolesResponse' description: Gets the available roles and their permissions security: - Auth query parameter: [] @@ -883,10 +781,7 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' description: "Changes the permissions for a user. Available values for each permission are: `default`, `yes`, and `no`. `default` will use the user's role's default permission." security: - Auth query parameter: [] @@ -894,18 +789,7 @@ paths: content: application/json: schema: - type: object - properties: - user_uid: - type: string - permission: - type: string - new_value: - type: string - required: - - user_uid - - permission - - new_value + $ref: '#/components/schemas/ChangeUserPermissionsRequest' tags: - multi-user mode /api/changeRolePermissions: @@ -918,26 +802,12 @@ paths: content: application/json: schema: - type: object - properties: - success: - type: boolean + $ref: '#/components/schemas/SuccessObject' requestBody: content: application/json: schema: - type: object - properties: - role: - type: string - permission: - type: string - new_value: - type: string - required: - - role - - permission - - new_value + $ref: '#/components/schemas/ChangeRolePermissionsRequest' description: 'Changes the permissions for a role. Available values for each permission are: `yes`, and `no`.' security: - Auth query parameter: [] @@ -953,20 +823,337 @@ paths: content: application/json: schema: - type: object - properties: - users: - type: array - items: - $ref: '#/components/schemas/user' + $ref: '#/components/schemas/GetUsersResponse' description: 'Gets all users, returns a list of the user objects including their user permissions, videos, playlists, subscriptions, etc.' security: - Auth query parameter: [] tags: - multi-user mode + /api/versionInfo: + get: + tags: + - server + summary: Gets server version info + operationId: get-api-versionInfo + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/VersionInfoResponse' + security: + - Auth query parameter: [] + /api/getLogs: + post: + summary: Gets logs from server + tags: + - server + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetLogsResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetLogsRequest' + operationId: post-api-getLogs + description: Gets logs from server + security: + - Auth query parameter: [] + /api/clearAllLogs: + post: + summary: Clears the log file + tags: + - server + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + operationId: post-api-clearAllLogs + description: Clears the log file + security: + - Auth query parameter: [] + /api/getDBInfo: + get: + tags: + - db + summary: Gets information on the DB + operationId: get-api-getDBInfo + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/DBInfoResponse' + security: + - Auth query parameter: [] + /api/transferDB: + post: + summary: Transfers DB between Local and MongoDB + tags: + - db + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TransferDBResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TransferDBRequest' + operationId: post-api-transferDB + description: Initiates a transfer between Local and MongoDB. Connection string must be set. + security: + - Auth query parameter: [] + /api/testConnectionString: + post: + summary: Tests a MongoDB connection string + tags: + - db + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TestConnectionStringResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TestConnectionStringRequest' + operationId: post-api-testConnectionString + description: Tests a MongoDB connection string and returns an error if one exists. + security: + - Auth query parameter: [] + /api/getFullTwitchChat: + post: + summary: Gets the downloaded Twitch Chat (VODs only) + tags: + - twitch + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetFullTwitchChatResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetFullTwitchChatRequest' + operationId: post-api-getFullTwitchChat + description: Gets the downloaded Twitch Chat (VODs only) + security: + - Auth query parameter: [] + /api/downloadTwitchChatByVODID: + post: + summary: Downloads Twitch Chat for a VOD + tags: + - twitch + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadTwitchChatByVODIDResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DownloadTwitchChatByVODIDRequest' + operationId: post-api-downloadTwitchChatByVODID + description: Downloads Twitch Chat for a VOD + security: + - Auth query parameter: [] + /api/checkConcurrentStream: + post: + summary: Checks status of a concurrent stream + tags: + - player + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/CheckConcurrentStreamResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CheckConcurrentStreamRequest' + operationId: post-api-checkConcurrentStream + description: Checks status of a concurrent stream + security: + - Auth query parameter: [] + /api/updateConcurrentStream: + post: + summary: Updates a concurrent stream + tags: + - player + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateConcurrentStreamResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateConcurrentStreamRequest' + operationId: post-api-updateConcurrentStream + description: Updates a concurrent stream + security: + - Auth query parameter: [] + /api/getAllCategories: + post: + summary: Gets all categories + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllCategoriesResponse' + operationId: post-api-getAllCategories + description: Gets all categories + security: + - Auth query parameter: [] + /api/createCategory: + post: + summary: Creates a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCategoryResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCategoryRequest' + operationId: post-api-createCategory + description: Creates a category + security: + - Auth query parameter: [] + /api/deleteCategory: + post: + summary: Deletes a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteCategoryRequest' + operationId: post-api-deleteCategory + description: Deletes a category + security: + - Auth query parameter: [] + /api/updateCategory: + post: + summary: Updates a category + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateCategoryRequest' + operationId: post-api-updateCategory + description: Updates a category + security: + - Auth query parameter: [] + /api/updateCategories: + post: + summary: Updates all categories + tags: + - categories + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateCategoriesRequest' + operationId: post-api-updateCategories + description: Updates all categories + security: + - Auth query parameter: [] components: schemas: - body: + SuccessObject: + required: + - success + type: object + properties: + success: + type: boolean + FileType: + type: string + enum: + - audio + - video + CropFileSettings: + type: object + required: + - cropFileStart + - cropFileEnd + properties: + cropFileStart: + type: number + cropFileEnd: + type: number + Config: + required: + - YoutubeDLMaterial + type: object + properties: + YoutubeDLMaterial: + type: object + DownloadRequest: required: - url type: object @@ -975,64 +1162,76 @@ components: type: string customQualityConfiguration: type: string + description: Video format code. Overrides other quality options. example: '251' - maxBitrate: - type: string - example: '160' customArgs: type: string + description: Custom command-line arguments for youtube-dl. Overrides all other options, except url. + additionalArgs: + type: string + description: Additional command-line arguments for youtube-dl. Added to whatever args would normally be used. customOutput: type: string + description: Custom output filename template. youtubeUsername: type: string + description: Login with this account ID youtubePassword: type: string - inline_response_200: - required: - - audiopathEncoded - - uid - type: object - properties: - uid: - type: string - file_names: - type: string - audiopathEncoded: - type: string - body_1: - required: - - url - type: object - properties: - url: - type: string - customQualityConfiguration: - type: string - example: 242+251 + description: Account password selectedHeight: type: string + description: Height of the video, if known example: '1080' - customArgs: + maxBitrate: type: string - customOutput: - type: string - youtubeUsername: - type: string - youtubePassword: - type: string - inline_response_200_1: - required: - - uid - - videopathEncoded + description: Specify ffmpeg/avconv audio quality + example: '160' + type: + $ref: '#/components/schemas/FileType' + cropFileSettings: + $ref: '#/components/schemas/CropFileSettings' + DownloadResponse: type: object properties: - uid: + download: + $ref: '#/components/schemas/Download' + GenerateArgsResponse: + type: object + properties: + args: + type: array + items: + type: string + GetDownloadRequest: + type: object + properties: + download_uid: type: string - file_names: - type: string - videopathEncoded: - type: string - inline_response_200_2: + required: + - download_uid + GetDownloadResponse: + type: object + properties: + download: + $ref: '#/components/schemas/Download' + GetAllDownloadsRequest: + type: object + properties: + uids: + type: array + items: + type: string + description: Filters downloads with the array + nullable: true + GetAllDownloadsResponse: + type: object + properties: + downloads: + type: array + items: + $ref: '#/components/schemas/Download' + GetMp3sResponse: required: - mp3s - playlists @@ -1041,81 +1240,78 @@ components: mp3s: type: array items: - $ref: '#/components/schemas/inline_response_200_2_mp3s' + $ref: '#/components/schemas/DatabaseFile' playlists: type: array description: All audio playlists items: - $ref: '#/components/schemas/inline_response_200_2_playlists' - inline_response_200_3: + $ref: '#/components/schemas/Playlist' + GetMp4sResponse: required: - mp4s + - playlists type: object properties: mp4s: type: array items: - $ref: '#/components/schemas/inline_response_200_3_mp4s' + $ref: '#/components/schemas/DatabaseFile' playlists: type: array description: All video playlists items: - type: object - body_2: + $ref: '#/components/schemas/Playlist' + GetAllFilesResponse: + required: + - files + - playlists + type: object + properties: + files: + type: array + items: + $ref: '#/components/schemas/DatabaseFile' + playlists: + type: array + description: All video playlists + items: + $ref: '#/components/schemas/Playlist' + GetFileRequest: required: - uid type: object properties: uid: type: string + description: Video UID type: + $ref: '#/components/schemas/FileType' + uuid: type: string - inline_response_200_4: + description: User UID + GetFileResponse: required: - - file - success type: object properties: success: - type: string + type: boolean file: - $ref: '#/components/schemas/inline_response_200_2_mp3s' - body_3: + $ref: '#/components/schemas/DatabaseFile' + SharingToggle: required: - - is_playlist - - type - uid type: object properties: uid: type: string - type: - type: string is_playlist: type: boolean - inline_response_200_5: - required: - - success - type: object - properties: - success: - type: boolean - body_4: - required: - - type - - uid - type: object - properties: - type: - type: string - uid: - type: string - description: uid is either the video uid or the playlist ID - is_playlist: - type: boolean - body_5: + SubscribeRequest: required: + - name - url + - streamingOnly type: object properties: name: @@ -1124,48 +1320,34 @@ components: type: string timerange: type: string - streamingOnly: + audioOnly: type: boolean - inline_response_200_6: + customArgs: + type: string + customFileOutput: + type: string + maxQuality: + type: string + SubscribeResponse: required: - new_sub type: object properties: new_sub: - $ref: '#/components/schemas/inline_response_200_6_new_sub' + $ref: '#/components/schemas/Subscription' error: type: string - apiunsubscribe_sub: - required: - - id - - name - - url - - videos - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - videos: - type: array - items: - type: object - body_6: + UnsubscribeRequest: required: - sub type: object properties: sub: - $ref: '#/components/schemas/apiunsubscribe_sub' + $ref: '#/components/schemas/SubscriptionRequestData' deleteMode: type: boolean description: Defaults to false - inline_response_200_7: + UnsubscribeResponse: required: - success type: object @@ -1174,30 +1356,7 @@ components: type: boolean error: type: string - apideleteSubscriptionFile_sub: - required: - - id - - isPlaylist - - name - - url - - videos - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - videos: - type: array - items: - type: object - body_7: + DeleteSubscriptionFileRequest: required: - file - sub @@ -1205,50 +1364,44 @@ components: properties: file: type: string + file_uid: + type: string sub: - $ref: '#/components/schemas/apideleteSubscriptionFile_sub' + $ref: '#/components/schemas/SubscriptionRequestData' deleteForever: type: boolean description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.' - inline_response_200_8: - type: object - properties: - success: - type: boolean - body_8: + GetSubscriptionRequest: required: - - subID + - id type: object properties: - subID: + id: type: string - inline_response_200_9: + description: Subscription ID + name: + type: string + description: Subscription name + GetSubscriptionResponse: required: - files - subscription type: object properties: subscription: - $ref: '#/components/schemas/inline_response_200_9_subscription' + $ref: '#/components/schemas/Subscription' files: type: array items: type: object - body_9: + DownloadVideosForSubscriptionRequest: required: - subID type: object properties: subID: type: string - inline_response_200_10: - required: - - success - type: object - properties: - success: - type: number - inline_response_200_11: + GetAllSubscriptionsResponse: required: - subscriptions type: object @@ -1256,10 +1409,10 @@ components: subscriptions: type: array items: - $ref: '#/components/schemas/inline_response_200_11_subscriptions' - body_10: + $ref: '#/components/schemas/Subscription' + CreatePlaylistRequest: required: - - fileNames + - uids - playlistName - thumbnailURL - type @@ -1267,35 +1420,38 @@ components: properties: playlistName: type: string - fileNames: + uids: type: array items: type: string type: - type: string + $ref: '#/components/schemas/FileType' thumbnailURL: type: string - inline_response_200_12: + CreatePlaylistResponse: required: - new_playlist - success type: object properties: new_playlist: - $ref: '#/components/schemas/inline_response_200_12_new_playlist' + $ref: '#/components/schemas/Playlist' success: type: boolean - body_11: + GetPlaylistRequest: required: - - playlistID - - type + - playlist_id type: object properties: - playlistID: + playlist_id: type: string type: + $ref: '#/components/schemas/FileType' + uuid: type: string - inline_response_200_13: + include_file_metadata: + type: boolean + GetPlaylistResponse: required: - playlist - success @@ -1303,93 +1459,66 @@ components: type: object properties: playlist: - $ref: '#/components/schemas/inline_response_200_2_playlists' + $ref: '#/components/schemas/Playlist' type: - type: string + $ref: '#/components/schemas/FileType' success: type: boolean - body_12: - required: - - fileNames - - playlistID - - type + GetPlaylistsRequest: type: object properties: - playlistID: - type: string - fileNames: + include_categories: + type: boolean + GetPlaylistsResponse: + required: + - playlists + type: object + properties: + playlists: type: array items: - type: string - type: - type: string - body_13: + $ref: '#/components/schemas/Playlist' + UpdatePlaylistRequest: required: - - playlistID + - playlist + type: object + properties: + playlist: + $ref: '#/components/schemas/Playlist' + DeletePlaylistRequest: + required: + - playlist_id - type type: object properties: - playlistID: + playlist_id: type: string type: - type: string - body_14: - required: - - uid + $ref: '#/components/schemas/FileType' + DownloadFileRequest: type: object properties: uid: type: string - body_15: - required: - - fileNames - - type - type: object - properties: - fileNames: - type: array - description: Array of 1 or more files to download - items: - type: string - zip_mode: - type: boolean - type: + uuid: type: string - outputName: + sub_id: type: string - fullPathProvided: + playlist_id: type: string - subscriptionName: - type: boolean - description: Only used for subscriptions - subscriptionPlaylist: - type: boolean - description: Only used for subscriptions - body_16: - required: - - fileName - - type - type: object - properties: - fileName: - type: string - type: - type: string - apidownloadArchive_sub: - required: - - archive_dir - type: object - properties: - archive_dir: - type: string - body_17: + DownloadArchiveRequest: required: - sub type: object properties: sub: - $ref: '#/components/schemas/apidownloadArchive_sub' - inline_response_200_14: + required: + - archive_dir + type: object + properties: + archive_dir: + type: string + UpdaterStatus: required: - details - updating @@ -1399,11 +1528,271 @@ components: type: boolean details: type: string - body_18: + error: + type: boolean + UpdateServerRequest: + required: + - tag type: object properties: tag: type: string + DBInfoResponse: + required: + - db_info + type: object + properties: + using_local_db: + type: boolean + stats_by_table: + type: object + properties: + files: + $ref: '#/components/schemas/TableInfo' + playlists: + $ref: '#/components/schemas/TableInfo' + categories: + $ref: '#/components/schemas/TableInfo' + subscriptions: + $ref: '#/components/schemas/TableInfo' + users: + $ref: '#/components/schemas/TableInfo' + roles: + $ref: '#/components/schemas/TableInfo' + download_queue: + $ref: '#/components/schemas/TableInfo' + TransferDBResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + TransferDBRequest: + required: + - local_to_remote + type: object + properties: + local_to_remote: + description: True if transfering DB from Local to MongoDB, false if transferring DB from MongoDB to Local + type: boolean + TestConnectionStringResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + TestConnectionStringRequest: + required: + - connection_string + type: object + properties: + connection_string: + description: MongoDB connection string + type: string + GetFullTwitchChatResponse: + required: + - success + type: object + properties: + success: + type: boolean + error: + type: string + GetFullTwitchChatRequest: + required: + - id + - type + type: object + properties: + id: + description: File ID + type: string + type: + $ref: '#/components/schemas/FileType' + uuid: + description: User UID + type: string + sub: + $ref: '#/components/schemas/Subscription' + DownloadTwitchChatByVODIDResponse: + required: + - chat + type: object + properties: + chat: + type: array + items: + $ref: '#/components/schemas/TwitchChatMessage' + DownloadTwitchChatByVODIDRequest: + required: + - id + - type + - vodId + type: object + properties: + id: + description: File ID + type: string + vodId: + description: ID of the VOD + type: string + type: + $ref: '#/components/schemas/FileType' + uuid: + description: User UID + type: string + sub: + $ref: '#/components/schemas/Subscription' + CheckConcurrentStreamResponse: + required: + - stream + type: object + properties: + stream: + $ref: '#/components/schemas/ConcurrentStream' + CheckConcurrentStreamRequest: + required: + - uid + type: object + properties: + uid: + description: UID of the concurrent stream + type: string + UpdateConcurrentStreamResponse: + required: + - stream + type: object + properties: + stream: + $ref: '#/components/schemas/ConcurrentStream' + UpdateConcurrentStreamRequest: + allOf: + - $ref: '#/components/schemas/ConcurrentStream' + - type: object + properties: + uid: + type: string + description: Concurrent stream UID + required: + - uid + GetLogsResponse: + type: object + properties: + logs: + type: string + description: Number of lines to retrieve from the bottom + success: + type: boolean + GetLogsRequest: + required: + - connection_string + type: object + properties: + lines: + type: number + GetFileFormatsRequest: + type: object + properties: + url: + type: string + GetFileFormatsResponse: + required: + - result + - success + type: object + properties: + success: + type: boolean + result: + allOf: + - $ref: '#/components/schemas/file' + - type: object + properties: + formats: + type: array + items: + type: object + IncrementViewCountRequest: + required: + - file_uid + type: object + properties: + file_uid: + type: string + sub_id: + type: string + uuid: + type: string + description: User UID + AddFileToPlaylistRequest: + required: + - playlist_id + - file_uid + type: object + properties: + file_uid: + type: string + playlist_id: + type: string + GetAllCategoriesResponse: + required: + - categories + type: object + properties: + categories: + type: array + items: + $ref: '#/components/schemas/Category' + CreateCategoryResponse: + type: object + properties: + new_category: + $ref: '#/components/schemas/Category' + success: + type: boolean + CreateCategoryRequest: + required: + - name + type: object + properties: + name: + type: string + DeleteCategoryRequest: + required: + - category_uid + type: object + properties: + category_uid: + type: string + UpdateCategoryRequest: + required: + - category + type: object + properties: + category: + $ref: '#/components/schemas/Category' + UpdateCategoriesRequest: + required: + - categories + type: object + properties: + categories: + type: array + items: + $ref: '#/components/schemas/Category' + VersionInfoResponse: + required: + - version_info + type: object + properties: + version_info: + $ref: '#/components/schemas/Version' inline_response_200_15: required: - is_set @@ -1425,7 +1814,7 @@ components: properties: unhashed_pin: type: string - inline_response_200_16: + GenerateNewApiKeyResponse: required: - new_api_key type: object @@ -1433,31 +1822,33 @@ components: new_api_key: type: string example: 4241b401-7236-493e-92b5-b72696b9d853 - body_21: + DeleteMp3Mp4Request: required: - uid type: object properties: uid: type: string - inline_response_200_17: + blacklistMode: + type: boolean + ConfigResponse: required: - config_file - success type: object properties: config_file: - type: object + $ref: '#/components/schemas/Config' success: type: boolean - body_22: + SetConfigRequest: required: - new_config_file type: object properties: new_config_file: - type: object - inline_response_200_2_mp3s: + $ref: '#/components/schemas/Config' + DatabaseFile: required: - duration - id @@ -1497,17 +1888,20 @@ components: type: string sharingEnabled: type: boolean - inline_response_200_2_playlists: + Playlist: required: - - fileNames + - uids - id - name - thumbnailURL + - type + - registered + - duration type: object properties: name: type: string - fileNames: + uids: type: array items: type: string @@ -1515,125 +1909,120 @@ components: type: string thumbnailURL: type: string - inline_response_200_3_mp4s: - required: - - duration - - id - - isAudio - - path - - size - - thumbnailURL - - title - - uid - - upload_date - - uploader - - url - type: object - properties: - id: - type: string - title: - type: string - thumbnailURL: - type: string - isAudio: - type: boolean + type: + $ref: '#/components/schemas/FileType' + registered: + type: number duration: type: number - url: - type: string - uploader: - type: string - size: - type: number - path: - type: string - upload_date: + user_uid: type: string + Download: + required: + - url + - type + - title + - options + - uid + - step_index + - paused + - running + - finished_step + - percent_complete + - finished + - timestamp_start + type: object + properties: uid: type: string - sharingEnabled: - type: boolean - inline_response_200_6_new_sub: - required: - - id - - name - - url - type: object - properties: - name: + ui_uid: type: string + running: + type: boolean + finished: + type: boolean + paused: + type: boolean + finished_step: + type: boolean url: type: string - id: - type: string - streamingOnly: - type: boolean - timerange: - type: string - inline_response_200_9_subscription: - required: - - archive - - id - - isPlaylist - - name - - url - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - archive: - type: string - inline_response_200_11_subscriptions: - required: - - archive - - id - - isPlaylist - - name - - streamingOnly - - url - type: object - properties: - name: - type: string - url: - type: string - id: - type: string - streamingOnly: - type: boolean - isPlaylist: - type: boolean - archive: - type: string - inline_response_200_12_new_playlist: - required: - - fileNames - - id - - name - - thumbnailURL - - type - type: object - properties: - name: - type: string - fileNames: - type: string - id: - type: string - thumbnailURL: - type: string type: type: string - user: + title: + type: string + step_index: + type: number + percent_complete: + type: number + timestamp_start: + type: number + error: + type: string + description: Error text, set if download fails. + nullable: true + user_uid: + type: string + sub_id: + type: string + sub_name: + type: string + SubscriptionRequestData: + required: + - id + - name + type: object + properties: + name: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + isPlaylist: + type: boolean + archive: + type: string + Subscription: + required: + - id + - name + - url + - type + - user_uid + - streamingOnly + - isPlaylist + - videos + type: object + properties: + name: + type: string + url: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + user_uid: + type: string + nullable: true + streamingOnly: + type: boolean + isPlaylist: + type: boolean + archive: + type: string + timerange: + type: string + custom_args: + type: string + custom_output: + type: string + videos: + type: array + items: + type: object + User: title: user type: object properties: @@ -1668,7 +2057,7 @@ components: subscriptions: type: array items: - $ref: '#/components/schemas/inline_response_200_9_subscription' + $ref: '#/components/schemas/Subscription' created: type: number role: @@ -1676,17 +2065,233 @@ components: permissions: type: array items: - type: string + $ref: '#/components/schemas/UserPermission' permission_overrides: type: array items: - type: string + $ref: '#/components/schemas/UserPermission' + UserPermission: + type: string + enum: + - filemanager + - settings + - subscriptions + - sharing + - advanced_download + - downloads_manager + YesNo: + type: string + enum: + - 'yes' + - 'no' + TableInfo: + type: object + properties: + records_count: + type: number + TwitchChatMessage: + type: object + properties: + created_at: + type: string + content_offset_seconds: + type: number + commenter: + type: object + properties: + name: + type: string + _id: + type: string + created_at: + type: string + message: + type: object + properties: + body: + type: string + user_color: + type: string + ConcurrentStream: + type: object + properties: + playback_timestamp: + type: number + unix_timestamp: + type: number + playing: + type: boolean + Category: + type: object + properties: + name: + type: string + uid: + type: string + rules: + type: array + items: + $ref: '#/components/schemas/CategoryRule' + custom_output: + type: string + description: Overrides file output for downloaded files in category + CategoryRule: + type: object + properties: + preceding_operator: + type: string + enum: + - 'or' + - 'and' + comparator: + type: string + enum: + - 'includes' + - 'not_includes' + - 'equals' + - 'not_equals' + Version: + type: object + properties: + type: + type: string + tag: + type: string + commit: + type: string + date: + type: string + BaseChangePermissionsRequest: + required: + - permission + - new_value + type: object + properties: + permission: + $ref: '#/components/schemas/UserPermission' + new_value: + $ref: '#/components/schemas/YesNo' + ChangeUserPermissionsRequest: + allOf: + - $ref: '#/components/schemas/BaseChangePermissionsRequest' + - type: object + properties: + user_uid: + type: string + required: + - user_uid + ChangeRolePermissionsRequest: + allOf: + - $ref: '#/components/schemas/BaseChangePermissionsRequest' + - type: object + properties: + role: + type: string + required: + - role file: title: file type: object properties: id: type: string + RegisterRequest: + required: + - userid + - username + - password + type: object + properties: + userid: + type: string + username: + type: string + password: + type: string + RegisterResponse: + type: object + properties: + user: + $ref: '#/components/schemas/User' + LoginRequest: + required: + - username + - password + type: object + properties: + username: + type: string + password: + type: string + LoginResponse: + type: object + properties: + user: + $ref: '#/components/schemas/User' + token: + type: string + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + available_permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + UpdateUserRequest: + required: + - change_object + type: object + properties: + change_object: + required: + - uid + type: object + properties: + uid: + type: string + name: + type: string + role: + type: string + DeleteUserRequest: + required: + - uid + type: object + properties: + uid: + type: string + GetUsersResponse: + required: + - users + type: object + properties: + users: + type: array + items: + $ref: '#/components/schemas/User' + GetRolesResponse: + required: + - roles + type: object + properties: + roles: + type: object + properties: + admin: + type: object + properties: + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' + user: + type: object + properties: + permissions: + type: array + items: + $ref: '#/components/schemas/UserPermission' securitySchemes: Auth query parameter: name: apiKey diff --git a/README.md b/README.md index c073161..94d09f1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker Debian/Ubuntu: ```bash -sudo apt-get install nodejs youtube-dl ffmpeg +sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm ``` CentOS 7: @@ -67,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend. -Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. +Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`. @@ -77,6 +77,10 @@ Alternatively, you can port forward the port specified in the config (defaults t ## Docker +### Host-specific instructions + +If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp) + ### Setup If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple. @@ -106,6 +110,12 @@ To get started, go to the settings menu and enable the public API from the *Extr Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above. +## iOS Shortcut + +If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?) + +You can download Shortcut [here.](https://routinehub.co/shortcut/10283/) + ## Contributing If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away. @@ -124,7 +134,7 @@ Official translators: * German - UnlimitedCookies * Chinese - TyRoyal -See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project. +See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project. ## License diff --git a/backend/.eslintrc.json b/backend/.eslintrc.json new file mode 100644 index 0000000..172a2e4 --- /dev/null +++ b/backend/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "node": true, + "es2021": true + }, + "extends": [ + "eslint:recommended" + ], + "parser": "esprima", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [], + "rules": { + }, + "root": true +} diff --git a/backend/app.js b/backend/app.js index 888ccf7..b12d502 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1,37 +1,36 @@ const { uuid } = require('uuidv4'); -var fs = require('fs-extra'); -var { promisify } = require('util'); -var auth_api = require('./authentication/auth'); -var winston = require('winston'); -var path = require('path'); -var youtubedl = require('youtube-dl'); -var ffmpeg = require('fluent-ffmpeg'); -var compression = require('compression'); -var glob = require("glob") -var multer = require('multer'); -var express = require("express"); -var bodyParser = require("body-parser"); -var archiver = require('archiver'); -var unzipper = require('unzipper'); -var db_api = require('./db'); -var utils = require('./utils') -var mergeFiles = require('merge-files'); +const fs = require('fs-extra'); +const { promisify } = require('util'); +const auth_api = require('./authentication/auth'); +const winston = require('winston'); +const path = require('path'); +const compression = require('compression'); +const multer = require('multer'); +const express = require("express"); +const bodyParser = require("body-parser"); +const archiver = require('archiver'); +const unzipper = require('unzipper'); +const db_api = require('./db'); +const utils = require('./utils') const low = require('lowdb') -var ProgressBar = require('progress'); -const NodeID3 = require('node-id3') -const downloader = require('youtube-dl/lib/downloader') +const ProgressBar = require('progress'); const fetch = require('node-fetch'); -var URL = require('url').URL; -const shortid = require('shortid') -const url_api = require('url'); -var config_api = require('./config.js'); -var subscriptions_api = require('./subscriptions') -var categories_api = require('./categories'); -var twitch_api = require('./twitch'); +const URL = require('url').URL; const CONSTS = require('./consts') -const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); -var ps = require('ps-node'); +const ps = require('ps-node'); + +// needed if bin/details somehow gets deleted +if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) + +const youtubedl = require('youtube-dl'); + +const logger = require('./logger'); +const config_api = require('./config.js'); +const downloader_api = require('./downloader'); +const subscriptions_api = require('./subscriptions'); +const categories_api = require('./categories'); +const twitch_api = require('./twitch'); const is_windows = process.platform === 'win32'; @@ -39,7 +38,6 @@ var app = express(); // database setup const FileSync = require('lowdb/adapters/FileSync'); -const config = require('./config.js'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -59,30 +57,12 @@ const admin_token = '4241b401-7236-493e-92b5-b72696b9d853'; // logging setup -// console format -const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => { - return `${timestamp} ${level.toUpperCase()}: ${message}`; -}); -const logger = winston.createLogger({ - level: 'info', - format: winston.format.combine(winston.format.timestamp(), defaultFormat), - defaultMeta: {}, - transports: [ - // - // - Write to all logs with level `info` and below to `combined.log` - // - Write all logs error (and below) to `error.log`. - // - new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), - new winston.transports.File({ filename: 'appdata/logs/combined.log' }), - new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'}) - ] -}); - -config_api.initialize(logger); -auth_api.initialize(db, users_db, logger); -db_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); -categories_api.initialize(db, users_db, logger, db_api); +config_api.initialize(); +db_api.initialize(db, users_db); +auth_api.initialize(db_api); +downloader_api.initialize(db_api); +subscriptions_api.initialize(db_api, downloader_api); +categories_api.initialize(db_api); // Set some defaults db.defaults( @@ -120,37 +100,33 @@ users_db.defaults( ).write(); // config values -var frontendUrl = null; -var backendUrl = null; -var backendPort = null; -var basePath = null; -var audioFolderPath = null; -var videoFolderPath = null; -var downloadOnlyMode = null; -var useDefaultDownloadingAgent = null; -var customDownloadingAgent = null; -var allowSubscriptions = null; -var subscriptionsCheckInterval = null; -var archivePath = path.join(__dirname, 'appdata', 'archives'); +let url = null; +let backendPort = null; +let useDefaultDownloadingAgent = null; +let customDownloadingAgent = null; +let allowSubscriptions = null; +let archivePath = path.join(__dirname, 'appdata', 'archives'); // other needed values -var url_domain = null; -var updaterStatus = null; +let url_domain = null; +let updaterStatus = null; -var timestamp_server_start = Date.now(); +const concurrentStreams = {}; if (debugMode) logger.info('YTDL-Material in debug mode!'); // check if just updated -const just_restarted = fs.existsSync('restart.json'); -if (just_restarted) { +const just_updated = fs.existsSync('restart_update.json'); +if (just_updated) { updaterStatus = { updating: false, details: 'Update complete! You are now on ' + CONSTS['CURRENT_VERSION'] } - fs.unlinkSync('restart.json'); + fs.unlinkSync('restart_update.json'); } +if (fs.existsSync('restart_general.json')) fs.unlinkSync('restart_general.json'); + // updates & starts youtubedl (commented out b/c of repo takedown) // startYoutubeDL(); @@ -166,6 +142,14 @@ var validDownloadingAgents = [ const subscription_timeouts = {}; +let version_info = null; +if (fs.existsSync('version.json')) { + version_info = fs.readJSONSync('version.json'); + logger.verbose(`Version info: ${JSON.stringify(version_info, null, 2)}`); +} else { + version_info = {'type': 'N/A', 'tag': 'N/A', 'commit': 'N/A', 'date': 'N/A'}; +} + // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); let writeConfigMode = process.env.write_ytdl_config; @@ -179,8 +163,6 @@ if (writeConfigMode) { loadConfig(); } -var downloads = {}; - app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); @@ -189,74 +171,35 @@ app.use(auth_api.passport.initialize()); // actual functions -/** - * setTimeout, but its a promise. - * @param {number} ms - */ -async function wait(ms) { - await new Promise(resolve => { - setTimeout(resolve, ms); - }); -} - async function checkMigrations() { - // 3.5->3.6 migration - const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); - - if (!files_to_db_migration_complete) { - logger.info('Beginning migration: 3.5->3.6+') - const success = await runFilesToDBMigration() - if (success) { logger.info('3.5->3.6+ migration complete!'); } - else { logger.error('Migration failed: 3.5->3.6+'); } - } - // 4.1->4.2 migration const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value(); if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); - success = success && await addMetadataPropertyToDB('view_count'); - success = success && await addMetadataPropertyToDB('description'); - success = success && await addMetadataPropertyToDB('height'); - success = success && await addMetadataPropertyToDB('abr'); + success = success && await db_api.addMetadataPropertyToDB('view_count'); + success = success && await db_api.addMetadataPropertyToDB('description'); + success = success && await db_api.addMetadataPropertyToDB('height'); + success = success && await db_api.addMetadataPropertyToDB('abr'); + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } - return true; -} - -async function runFilesToDBMigration() { - try { - let mp3s = await getMp3s(); - let mp4s = await getMp4s(); - - for (let i = 0; i < mp3s.length; i++) { - let file_obj = mp3s[i]; - const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); - } - } - - for (let i = 0; i < mp4s.length; i++) { - let file_obj = mp4s[i]; - const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp4', 'video'); - } - } + const new_db_system_migration_complete = db.get('new_db_system_migration_complete').value(); + if (!new_db_system_migration_complete) { + logger.info('Beginning migration: 4.2->4.3+') + let success = await db_api.importJSONToDB(db.value(), users_db.value()); // sets migration to complete - db.set('files_to_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; + db.set('new_db_system_migration_complete', true).write(); + if (success) { logger.info('4.2->4.3+ migration complete!'); } + else { logger.error('Migration failed: 4.2->4.3+'); } } + + return true; } async function simplifyDBFileStructure() { @@ -296,28 +239,6 @@ async function simplifyDBFileStructure() { return true; } -async function addMetadataPropertyToDB(property_key) { - try { - const dirs_to_check = db_api.getFileDirectoriesAndDBs(); - for (const dir_to_check of dirs_to_check) { - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); - for (const file of files) { - if (file[property_key]) { - dir_to_check.dbPath.find({id: file.id}).assign({[property_key]: file[property_key]}).write(); - } - } - } - - // sets migration to complete - db.set('simplified_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; - } -} - async function startServer() { if (process.env.USING_HEROKU && process.env.PORT) { // default to heroku port if using heroku @@ -332,18 +253,12 @@ async function startServer() { }); } -async function restartServer() { - const restartProcess = () => { - spawn('node', ['app.js'], { - detached: true, - stdio: 'inherit' - }).unref() - process.exit() - } - logger.info('Update complete! Restarting server...'); +async function restartServer(is_update = false) { + logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`); // the following line restarts the server through nodemon - fs.writeFileSync('restart.json', 'internal use only'); + fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only'); + process.exit(1); } async function updateServer(tag) { @@ -386,8 +301,9 @@ async function updateServer(tag) { updating: true, 'details': 'Update complete! Restarting server...' } - restartServer(); + restartServer(true); }, err => { + logger.error(err); updaterStatus = { updating: false, error: true, @@ -418,12 +334,10 @@ async function downloadReleaseFiles(tag) { fs.createReadStream(path.join(__dirname, `youtubedl-material-release-${tag}.zip`)).pipe(unzipper.Parse()) .on('entry', function (entry) { var fileName = entry.path; - var type = entry.type; // 'Directory' or 'File' - var size = entry.size; var is_dir = fileName.substring(fileName.length-1, fileName.length) === '/' if (!is_dir && fileName.includes('youtubedl-material/public/')) { // get public folder files - var actualFileName = fileName.replace('youtubedl-material/public/', ''); + const actualFileName = fileName.replace('youtubedl-material/public/', ''); if (actualFileName.length !== 0 && actualFileName.substring(actualFileName.length-1, actualFileName.length) !== '/') { fs.ensureDirSync(path.join(__dirname, 'public', path.dirname(actualFileName))); entry.pipe(fs.createWriteStream(path.join(__dirname, 'public', actualFileName))); @@ -432,7 +346,7 @@ async function downloadReleaseFiles(tag) { } } else if (!is_dir && !replace_ignore_list.includes(fileName)) { // get package.json - var actualFileName = fileName.replace('youtubedl-material/', ''); + const actualFileName = fileName.replace('youtubedl-material/', ''); logger.verbose('Downloading file ' + actualFileName); entry.pipe(fs.createWriteStream(path.join(__dirname, actualFileName))); } else { @@ -531,7 +445,7 @@ async function backupServerLite() { }); // wait a tiny bit for the zip to reload in fs - await wait(100); + await utils.wait(100); return true; } @@ -561,9 +475,10 @@ async function getLatestVersion() { async function killAllDownloads() { const lookupAsync = promisify(ps.lookup); + let resultList = null; try { - await lookupAsync({ + resultList = await lookupAsync({ command: 'youtube-dl' }); } catch (err) { @@ -599,7 +514,7 @@ async function killAllDownloads() { async function setPortItemFromENV() { config_api.setConfigItem('ytdl_port', backendPort.toString()); - await wait(100); + await utils.wait(100); return true; } @@ -613,7 +528,7 @@ async function setConfigFromEnv() { let success = config_api.setConfigItems(config_items); if (success) { logger.info('Config items set using ENV variables.'); - await wait(100); + await utils.wait(100); return true; } else { logger.error('ERROR: Failed to set config items using ENV variables.'); @@ -624,6 +539,11 @@ async function setConfigFromEnv() { async function loadConfig() { loadConfigValues(); + // connect to DB + await db_api.connectToDB(); + db_api.database_initialized = true; + db_api.database_initialized_bs.next(true); + // creates archive path if missing await fs.ensureDir(archivePath); @@ -636,20 +556,20 @@ async function loadConfig() { // get subscriptions if (allowSubscriptions) { // set downloading to false - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds - watchSubscriptions(); - setInterval(() => { + const watchSubscriptionsInterval = function() { watchSubscriptions(); - }, subscriptionsCheckInterval * 1000); + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); + setTimeout(watchSubscriptionsInterval, subscriptionsCheckInterval*1000); + } + + watchSubscriptionsInterval(); } db_api.importUnregisteredFiles(); - // load in previous downloads - downloads = db.get('downloads').value(); - // start the server here startServer(); @@ -659,13 +579,9 @@ async function loadConfig() { function loadConfigValues() { url = !debugMode ? config_api.getConfigItem('ytdl_url') : 'http://localhost:4200'; backendPort = config_api.getConfigItem('ytdl_port'); - audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); - videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode'); useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent'); customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent'); allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions'); - subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { logger.info(`Using non-default downloading agent \'${customDownloadingAgent}\'`) @@ -690,13 +606,14 @@ function loadConfigValues() { function calculateSubcriptionRetrievalDelay(subscriptions_amount) { // frequency is once every 5 mins by default + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); let interval_in_ms = subscriptionsCheckInterval * 1000; const subinterval_in_ms = interval_in_ms/subscriptions_amount; return subinterval_in_ms; } async function watchSubscriptions() { - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = await subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; @@ -734,6 +651,7 @@ async function watchSubscriptions() { }, current_delay); subscription_timeouts[sub.id] = true; current_delay += delay_interval; + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; } } @@ -763,818 +681,17 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -function getThumbnailMp3(name) -{ - var obj = utils.getJSONMp3(name, audioFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getThumbnailMp4(name) -{ - var obj = utils.getJSONMp4(name, videoFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getFileSizeMp3(name) -{ - var jsonPath = audioFolderPath+name+".mp3.info.json"; - - if (fs.existsSync(jsonPath)) - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - else - var obj = 0; - - return obj.filesize; -} - -function getFileSizeMp4(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - var filesize = 0; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - for (i = 0; i < obj.formats.length; i++) - { - if (obj.formats[i].format_id == format) - { - filesize = obj.formats[i].filesize; - } - } - } - - return filesize; -} - -function getAmountDownloadedMp3(name) -{ - var partPath = audioFolderPath+name+".mp3.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - - - -function getAmountDownloadedMp4(name) -{ - var format = getVideoFormatID(name); - var partPath = videoFolderPath+name+".f"+format+".mp4.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - -function getVideoFormatID(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - return format; - } -} - -async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { - let zipFolderPath = null; - - if (!fullPathProvided) { - zipFolderPath = (type === 'audio') ? audioFolderPath : videoFolderPath - if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); - } else { - zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - - let ext = (type === 'audio') ? '.mp3' : '.mp4'; - - let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip')); - - var archive = archiver('zip', { - gzip: true, - zlib: { level: 9 } // Sets the compression level. - }); - - archive.on('error', function(err) { - logger.error(err); - throw err; - }); - - // pipe archive data to the output file - archive.pipe(output); - - for (let i = 0; i < fileNames.length; i++) { - let fileName = fileNames[i]; - let fileNamePathRemoved = path.parse(fileName).base; - let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName; - archive.file(file_path, {name: fileNamePathRemoved + ext}) - } - - await archive.finalize(); - - // wait a tiny bit for the zip to reload in fs - await wait(100); - return path.join(zipFolderPath,outputName + '.zip'); -} - -async function deleteAudioFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : audioFolderPath; - - var jsonPath = path.join(filePath,name+'.mp3.info.json'); - var altJSONPath = path.join(filePath,name+'.info.json'); - var audioFilePath = path.join(filePath,name+'.mp3'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - altJSONPath = path.join(__dirname, altJSONPath); - audioFilePath = path.join(__dirname, audioFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - let audioFileExists = await fs.pathExists(audioFilePath); - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_audio.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp3(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('audio', line); - } else { - logger.info('Could not find archive file for audio files. Creating...'); - await fs.close(await fs.open(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (audioFileExists) { - await fs.unlink(audioFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(audioFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -async function deleteVideoFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : videoFolderPath; - var jsonPath = path.join(filePath,name+'.info.json'); - - var altJSONPath = path.join(filePath,name+'.mp4.info.json'); - var videoFilePath = path.join(filePath,name+'.mp4'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - videoFilePath = path.join(__dirname, videoFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let videoFileExists = await fs.pathExists(videoFilePath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp4(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('video', line); - } else { - logger.info('Could not find archive file for videos. Creating...'); - fs.closeSync(fs.openSync(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (videoFileExists) { - await fs.unlink(videoFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(videoFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -/** - * @param {'audio' | 'video'} type - * @param {string[]} fileNames - */ -async function getAudioOrVideoInfos(type, fileNames) { - let result = await Promise.all(fileNames.map(async fileName => { - let fileLocation = videoFolderPath+fileName; - if (type === 'audio') { - fileLocation += '.mp3.info.json'; - } else if (type === 'video') { - fileLocation += '.info.json'; - } - - if (await fs.pathExists(fileLocation)) { - let data = await fs.readFile(fileLocation); - try { - return JSON.parse(data); - } catch (e) { - let suffix; - if (type === 'audio') { - suffix += '.mp3'; - } else if (type === 'video') { - suffix += '.mp4'; - } - - logger.error(`Could not find info for file ${fileName}${suffix}`); - } - } - return null; - })); - - return result.filter(data => data != null); -} - -// downloads - -async function downloadFileByURL_exec(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - - // audio / video specific vars - var is_audio = type === 'audio'; - var ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; - let category = null; - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'exec'; - let downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now(), - filesize: null - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - let download_checker = null; - - // get video info prior to download - let info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info && url.includes('youtu')) { - resolve(false); - return; - } else if (info) { - // check if it fits into a category. If so, then get info again using new downloadConfig - if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); - - // set custom output if the category has one and re-retrieve info so the download manager has the right file name - if (category && category['custom_output']) { - options.customOutput = category['custom_output']; - options.noRelativePath = true; - downloadConfig = await generateArgs(url, type, options); - info = await getVideoInfoByURL(url, downloadConfig, download); - } - - // store info in download for future use - if (Array.isArray(info)) { - download['fileNames'] = []; - for (let info_obj of info) download['fileNames'].push(info_obj['_filename']); - } else { - download['_filename'] = info['_filename']; - } - download['filesize'] = utils.getExpectedFileSize(info); - download_checker = setInterval(() => checkDownloadPercent(download), 1000); - } - - // download file - youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { - if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) - - download['downloading'] = false; - download['timestamp_end'] = Date.now(); - var file_uid = null; - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); - if (err) { - logger.error(err.stderr); - - download['error'] = err.stderr; - updateDownloads(); - resolve(false); - return; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { - download['error'] = 'No output. Check if video already exists in your archive.'; - logger.warn(`No output received for video download, check if it exists in your archive.`) - updateDownloads(); - - resolve(false); - return; - } - var file_names = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - - if (!output_json) { - continue; - } - - // get filepath with no extension - const filepath_no_extension = utils.removeFileExtension(output_json['_filename']); - - var full_file_path = filepath_no_extension + ext; - var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); - - if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 - && config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) { - let vodId = url.split('twitch.tv/videos/')[1]; - vodId = vodId.split('?')[0]; - twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user); - } - - // renames file if necessary due to bug - if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { - try { - fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); - logger.info('Renamed ' + file_name + '.webm to ' + file_name); - } catch(e) { - } - } - - if (type === 'audio') { - let tags = { - title: output_json['title'], - artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] - } - let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3'); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); - } - - const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); - const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; - - if (options.cropFileSettings) { - await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext); - } - - // registers file in DB - file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings); - - if (file_name) file_names.push(file_name); - } - - let is_playlist = file_names.length > 1; - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - download['complete'] = true; - download['fileNames'] = is_playlist ? file_names : [full_file_path] - updateDownloads(); - - var videopathEncoded = encodeURIComponent(file_names[0]); - - resolve({ - [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: is_playlist ? file_names : null, - uid: file_uid - }); - } - }); - }); -} - -async function downloadFileByURL_normal(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - var file_uid = null; - const is_audio = type === 'audio'; - const ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; } - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'normal'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - const video = youtubedl(url, - // Optional arguments passed to youtube-dl. - downloadConfig, - // Additional options can be given for calling `child_process.execFile()`. - { cwd: __dirname }); - - let video_info = null; - let file_size = 0; - - // Will be called when the download starts. - video.on('info', function(info) { - video_info = info; - file_size = video_info.size; - const json_path = utils.removeFileExtension(video_info._filename) + '.info.json'; - fs.ensureFileSync(json_path); - fs.writeJSONSync(json_path, video_info); - video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) - }); - // Will be called if download was already completed and there is nothing more to download. - video.on('complete', function complete(info) { - 'use strict' - logger.info('file ' + info._filename + ' already downloaded.') - }) - - let download_pos = 0; - video.on('data', function data(chunk) { - download_pos += chunk.length - // `size` should not be 0 here. - if (file_size) { - let percent = (download_pos / file_size * 100).toFixed(2) - download['percent_complete'] = percent; - } - }); - - video.on('end', async function() { - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - download['timestamp_end'] = Date.now(); - download['fileNames'] = [utils.removeFileExtension(video_info._filename) + ext]; - download['complete'] = true; - updateDownloads(); - - // audio-only cleanup - if (is_audio) { - // filename fix - video_info['_filename'] = utils.removeFileExtension(video_info['_filename']) + '.mp3'; - - // ID3 tagging - let tags = { - title: video_info['title'], - artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] - } - let success = NodeID3.write(tags, video_info._filename); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); - - const possible_webm_path = utils.removeFileExtension(video_info['_filename']) + '.webm'; - const possible_mp4_path = utils.removeFileExtension(video_info['_filename']) + '.mp4'; - // check if audio file is webm - if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']); - else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']); - } - - // registers file in DB - const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name)); - - resolve({ - [is_audio ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready - uid: file_uid - }); - }); - - video.on('error', function error(err) { - logger.error(err); - - download[error] = err; - updateDownloads(); - - resolve(false); - }); - }); - -} - -async function generateArgs(url, type, options) { - var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; - var globalArgs = config_api.getConfigItem('ytdl_custom_args'); - let useCookies = config_api.getConfigItem('ytdl_use_cookies'); - var is_audio = type === 'audio'; - - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath; - - var customArgs = options.customArgs; - var customOutput = options.customOutput; - var customQualityConfiguration = options.customQualityConfiguration; - - // video-specific args - var selectedHeight = options.selectedHeight; - - // audio-specific args - var maxBitrate = options.maxBitrate; - - var youtubeUsername = options.youtubeUsername; - var youtubePassword = options.youtubePassword; - - let downloadConfig = null; - let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4']; - const is_youtube = url.includes('youtu'); - if (!is_audio && !is_youtube) { - // tiktok videos fail when using the default format - qualityPath = null; - } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) { - qualityPath = ['-f', 'bestvideo+bestaudio'] - } - - if (customArgs) { - downloadConfig = customArgs.split(',,'); - } else { - if (customQualityConfiguration) { - qualityPath = ['-f', customQualityConfiguration]; - } else if (selectedHeight && selectedHeight !== '' && !is_audio) { - qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; - } else if (maxBitrate && is_audio) { - qualityPath = ['--audio-quality', maxBitrate] - } - - if (customOutput) { - customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); - downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; - } else { - downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; - } - - if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath); - - if (is_audio && !options.skip_audio_args) { - downloadConfig.push('-x'); - downloadConfig.push('--audio-format', 'mp3'); - } - - if (youtubeUsername && youtubePassword) { - downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); - } - - if (useCookies) { - if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) { - downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); - } else { - logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); - } - } - - if (!useDefaultDownloadingAgent && customDownloadingAgent) { - downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath; - const archive_path = path.join(archive_folder, `archive_${type}.txt`); - - await fs.ensureDir(archive_folder); - - // create archive file if it doesn't exist - if (!(await fs.pathExists(archive_path))) { - await fs.close(await fs.open(archive_path, 'w')); - } - - let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); - // create blacklist file if it doesn't exist - if (!(await fs.pathExists(blacklist_path))) { - await fs.close(await fs.open(blacklist_path, 'w')); - } - - let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); - await fs.ensureFile(merged_path); - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - let status = await mergeFiles(inputPathList, merged_path); - - options.merged_string = await fs.readFile(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - downloadConfig.push('--write-thumbnail'); - } - - if (globalArgs && globalArgs !== '') { - // adds global args - if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) { - // if global args has an output, replce the original output with that of global args - const original_output_index = downloadConfig.indexOf('-o'); - downloadConfig.splice(original_output_index, 2); - } - downloadConfig = downloadConfig.concat(globalArgs.split(',,')); - } - - } - - // filter out incompatible args - downloadConfig = filterArgs(downloadConfig, is_audio); - - logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); - return downloadConfig; -} - -async function getVideoInfoByURL(url, args = [], download = null) { - return new Promise(resolve => { - // remove bad args - const new_args = [...args]; - - const archiveArgIndex = new_args.indexOf('--download-archive'); - if (archiveArgIndex !== -1) { - new_args.splice(archiveArgIndex, 2); - } - - // actually get info - youtubedl.getInfo(url, new_args, (err, output) => { - if (output) { - resolve(output); - } else { - logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`); - if (err.stderr) { - logger.error(`${err.stderr}`) - } - if (download) { - download['error'] = `Failed pre-check for video info: ${err}`; - updateDownloads(); - } - resolve(null); - } - }); - }); -} - -function filterArgs(args, isAudio) { - const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs']; - const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail']; - const args_to_remove = isAudio ? video_only_args : audio_only_args; - return args.filter(x => !args_to_remove.includes(x)); -} - // currently only works for single urls -async function getUrlInfos(urls) { +async function getUrlInfos(url) { let startDate = Date.now(); let result = []; return new Promise(resolve => { - youtubedl.exec(urls.join(' '), ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { + youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { let new_date = Date.now(); let difference = (new_date - startDate)/1000; logger.debug(`URL info retrieval delay: ${difference} seconds.`); if (err) { - logger.error(`Error during parsing: ${err}`); + logger.error(`Error during retrieving formats for ${url}: ${err}`); resolve(null); } let try_putput = null; @@ -1582,107 +699,13 @@ async function getUrlInfos(urls) { try_putput = JSON.parse(output); result = try_putput; } catch(e) { - // probably multiple urls - logger.error('failed to parse for urls starting with ' + urls[0]); - // logger.info(output); + logger.error(`Failed to retrieve available formats for url: ${url}`); } resolve(result); }); }); } -// ffmpeg helper functions - -async function convertFileToMp3(input_file, output_file) { - logger.verbose(`Converting ${input_file} to ${output_file}...`); - return new Promise(resolve => { - ffmpeg(input_file).noVideo().toFormat('mp3') - .on('end', () => { - logger.verbose(`Conversion for '${output_file}' complete.`); - fs.unlinkSync(input_file) - resolve(true); - }) - .on('error', (err) => { - logger.error('Failed to convert audio file to the correct format.'); - logger.error(err); - resolve(false); - }).save(output_file); - }); -} - -async function cropFile(file_path, start, end, ext) { - return new Promise(resolve => { - const temp_file_path = `${file_path}.cropped${ext}`; - let base_ffmpeg_call = ffmpeg(file_path); - if (start) { - base_ffmpeg_call = base_ffmpeg_call.seekOutput(start); - } - if (end) { - base_ffmpeg_call = base_ffmpeg_call.duration(end - start); - } - base_ffmpeg_call - .on('end', () => { - logger.verbose(`Cropping for '${file_path}' complete.`); - fs.unlinkSync(file_path); - fs.moveSync(temp_file_path, file_path); - resolve(true); - }) - .on('error', (err, test, test2) => { - logger.error(`Failed to crop ${file_path}.`); - logger.error(err); - resolve(false); - }).save(temp_file_path); - }); -} - -// archive helper functions - -async function writeToBlacklist(type, line) { - let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); - // adds newline to the beginning of the line - line = '\n' + line; - await fs.appendFile(blacklistPath, line); -} - -// download management functions - -function updateDownloads() { - db.assign({downloads: downloads}).write(); -} - -function checkDownloadPercent(download) { - /* - This is more of an art than a science, we're just selecting files that start with the file name, - thus capturing the parts being downloaded in files named like so: '