diff --git a/browserslist b/.browserslistrc similarity index 100% rename from browserslist rename to .browserslistrc diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..c5624c4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,20 @@ +{ + "env": { + "browser": true, + "es2021": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [ + "@typescript-eslint" + ], + "rules": { + } +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..79e05a2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** +- YoutubeDL-Material version +- Docker tag: (optional) + +**Additional context** +Add any other context about the problem here. For example, a YouTube link. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d233fde --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]" +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..50f4eb5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,111 @@ +name: continuous integration + +on: + push: + branches: [master, feat/*] + tags: + - v* + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: setup node + uses: actions/setup-node@v2 + with: + node-version: '12' + cache: 'npm' + - name: install dependencies + run: | + npm install + cd backend + npm install + sudo npm install -g @angular/cli + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + - name: build + run: npm run build + - name: prepare artifact upload + shell: pwsh + run: | + New-Item -Name build -ItemType Directory + New-Item -Path build -Name youtubedl-material -ItemType Directory + Copy-Item -Path ./backend/appdata -Recurse -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/audio -Recurse -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/authentication -Recurse -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material + New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory + Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material + Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material + - name: upload build artifact + uses: actions/upload-artifact@v1 + with: + name: youtubedl-material + path: build + release: + runs-on: ubuntu-latest + needs: build + if: contains(github.ref, '/tags/v') + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: YoutubeDL-Material ${{ github.ref }} + body: | + # New features + # Minor additions + # Bug fixes + draft: true + prerelease: false + - name: download build artifact + uses: actions/download-artifact@v1 + with: + name: youtubedl-material + path: ${{runner.temp}}/youtubedl-material + - name: extract tag name + id: tag_name + run: echo ::set-output name=tag_name::${GITHUB_REF#refs/tags/} + - name: prepare release asset + shell: pwsh + run: Compress-Archive -Path ${{runner.temp}}/youtubedl-material -DestinationPath youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip + - name: upload release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip + asset_name: youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip + asset_content_type: application/zip + - name: upload docker-compose asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./docker-compose.yml + asset_name: docker-compose.yml + asset_content_type: text/plain diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..3063a97 --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,45 @@ +name: docker-release + +on: + workflow_dispatch: + inputs: + tags: + description: 'Docker tags' + required: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + - name: setup platform emulator + uses: docker/setup-qemu-action@v1 + - name: setup multi-arch docker build + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: build & push images + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm,linux/arm64/v8 + push: true + tags: ${{ github.event.inputs.tags }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..3fd4ab2 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,42 @@ +name: docker + +on: + push: + branches: [master] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: Set hash + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "version.json" + json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}' + dir: 'backend/' + - name: setup platform emulator + uses: docker/setup-qemu-action@v1 + - name: setup multi-arch docker build + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: build & push images + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm,linux/arm64/v8 + push: true + tags: tzahi12345/youtubedl-material:nightly diff --git a/.gitignore b/.gitignore index c99f53f..44fa9c2 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ backend/appdata/users.json backend/users/* backend/appdata/cookies.txt backend/public +src/assets/i18n/*.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..6db05be --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,25 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "start", + "problemMatcher": [], + "label": "Dev: start frontend", + "detail": "ng serve" + }, + { + "label": "Dev: start backend", + "type": "shell", + "command": "set YTDL_MODE=debug && node app.js", + "options": { + "cwd": "./backend" + }, + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [] + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 75b22d3..7b7d6cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.12 as frontend +FROM alpine:latest as frontend RUN apk add --no-cache \ npm @@ -11,28 +11,32 @@ RUN npm install COPY [ "angular.json", "tsconfig.json", "/build/" ] COPY [ "src/", "/build/src/" ] -RUN ng build --prod +RUN npm run build #--------------# -FROM alpine:3.12 +FROM alpine:latest ENV UID=1000 \ GID=1000 \ USER=youtube +ENV NO_UPDATE_NOTIFIER=true + RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID RUN apk add --no-cache \ ffmpeg \ npm \ python2 \ + python3 \ su-exec \ && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \ atomicparsley WORKDIR /app COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ] +RUN npm install pm2 -g RUN npm install && chown -R $UID:$GID ./ COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] @@ -40,4 +44,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ] EXPOSE 17442 ENTRYPOINT [ "/app/entrypoint.sh" ] -CMD [ "node", "app.js" ] +CMD [ "pm2-runtime", "pm2.config.js" ] diff --git a/Public API v1.yaml b/Public API v1.yaml index 866493e..5ebe9f3 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -282,12 +282,12 @@ paths: $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] - /api/getAllSubscriptions: + /api/getSubscriptions: post: tags: - subscriptions summary: Get all subscriptions - operationId: post-api-getAllSubscriptions + operationId: post-api-getSubscriptions requestBody: content: application/json: diff --git a/README.md b/README.md index 1c22952..a081a6e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # YoutubeDL-Material -[![](https://img.shields.io/docker/pulls/tzahi12345/youtubedl-material.svg)](https://hub.docker.com/r/tzahi12345/youtubedl-material) -[![](https://img.shields.io/docker/image-size/tzahi12345/youtubedl-material?sort=date)](https://hub.docker.com/r/tzahi12345/youtubedl-material) -[![](https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg)](https://heroku.com/deploy?template=https://github.com/Tzahi12345/YoutubeDL-Material) -[![](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues) -[![](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md) +[![Docker pulls badge](https://img.shields.io/docker/pulls/tzahi12345/youtubedl-material.svg)](https://hub.docker.com/r/tzahi12345/youtubedl-material) +[![Docker image size badge](https://img.shields.io/docker/image-size/tzahi12345/youtubedl-material?sort=date)](https://hub.docker.com/r/tzahi12345/youtubedl-material) +[![Heroku deploy badge](https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg)](https://heroku.com/deploy?template=https://github.com/Tzahi12345/YoutubeDL-Material) +[![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues) +[![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md) -YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 9](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend. +YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend. Now with [Docker](#Docker) support! @@ -16,27 +16,35 @@ Check out the prerequisites, and go to the installation section. Easy as pie! Here's an image of what it'll look like once you're done: -![frontpage](https://i.imgur.com/w8iofbb.png) - -With optional file management enabled (default): - -![frontpage_with_files](https://i.imgur.com/FTATqBM.png) + Dark mode: -![dark_mode](https://i.imgur.com/r5ZtBqd.png) + ### Prerequisites NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide. -Make sure you have these dependencies installed on your system: nodejs and youtube-dl. If you don't, run this command: +Debian/Ubuntu: +```bash +sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm ``` -sudo apt-get install nodejs youtube-dl + +CentOS 7: + +```bash +sudo yum install epel-release +sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm +sudo yum install centos-release-scl-rh +sudo yum install rh-nodejs12 +scl enable rh-nodejs12 bash +sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel ``` Optional dependencies: + * AtomicParsley (for embedding thumbnails, package name `atomicparsley`) ### Installing @@ -59,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend. -Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. +Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder. The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`. @@ -69,20 +77,26 @@ Alternatively, you can port forward the port specified in the config (defaults t ## Docker +### Host-specific instructions + +If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp) + ### Setup If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple. 1. Run `curl -L https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest/download/docker-compose.yml -o docker-compose.yml` to download the latest Docker Compose, or go to the [releases](https://github.com/Tzahi12345/YoutubeDL-Material/releases/) page to grab the version you'd like. 2. Run `docker-compose pull`. This will download the official YoutubeDL-Material docker image. -3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 8998" or something similar. -4. Make sure you can connect to the specified URL + port, and if so, you are done! +3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**. +4. Make sure you can connect to the specified URL + *external* port, and if so, you are done! + +NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`. ### Custom UID/GID By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so: -``` +```yml environment: UID: YOUR_UID GID: YOUR_GID @@ -109,11 +123,12 @@ If you're interested in translating the app into a new language, check out the [ * **Isaac Grynsztein** (me!) - *Initial work* Official translators: + * Spanish - tzahi12345 * German - UnlimitedCookies * Chinese - TyRoyal -See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project. +See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project. ## License diff --git a/angular.json b/angular.json index c32139b..46e84ea 100644 --- a/angular.json +++ b/angular.json @@ -45,8 +45,6 @@ ], "optimization": true, "outputHashing": "all", - "sourceMap": false, - "extractCss": true, "namedChunks": false, "aot": true, "extractLicenses": true, diff --git a/backend/.eslintrc.json b/backend/.eslintrc.json new file mode 100644 index 0000000..172a2e4 --- /dev/null +++ b/backend/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "node": true, + "es2021": true + }, + "extends": [ + "eslint:recommended" + ], + "parser": "esprima", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [], + "rules": { + }, + "root": true +} diff --git a/backend/app.js b/backend/app.js index d58e350..97a29df 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1,42 +1,43 @@ const { uuid } = require('uuidv4'); -var fs = require('fs-extra'); -var { promisify } = require('util'); -var auth_api = require('./authentication/auth'); -var winston = require('winston'); -var path = require('path'); -var youtubedl = require('youtube-dl'); -var ffmpeg = require('fluent-ffmpeg'); -var compression = require('compression'); -var glob = require("glob") -var multer = require('multer'); -var express = require("express"); -var bodyParser = require("body-parser"); -var archiver = require('archiver'); -var unzipper = require('unzipper'); -var db_api = require('./db') -var utils = require('./utils') -var mergeFiles = require('merge-files'); +const fs = require('fs-extra'); +const { promisify } = require('util'); +const auth_api = require('./authentication/auth'); +const winston = require('winston'); +const path = require('path'); +const compression = require('compression'); +const multer = require('multer'); +const express = require("express"); +const bodyParser = require("body-parser"); +const archiver = require('archiver'); +const unzipper = require('unzipper'); +const db_api = require('./db'); +const utils = require('./utils') const low = require('lowdb') -var ProgressBar = require('progress'); -const NodeID3 = require('node-id3') -const downloader = require('youtube-dl/lib/downloader') +const ProgressBar = require('progress'); const fetch = require('node-fetch'); -var URL = require('url').URL; -const shortid = require('shortid') -const url_api = require('url'); -var config_api = require('./config.js'); -var subscriptions_api = require('./subscriptions') +const URL = require('url').URL; const CONSTS = require('./consts') -const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); -var ps = require('ps-node'); +const ps = require('ps-node'); + +// needed if bin/details somehow gets deleted +if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) + +const youtubedl = require('youtube-dl'); + +const logger = require('./logger'); +const config_api = require('./config.js'); +const downloader_api = require('./downloader'); +const subscriptions_api = require('./subscriptions'); +const categories_api = require('./categories'); +const twitch_api = require('./twitch'); const is_windows = process.platform === 'win32'; var app = express(); // database setup -const FileSync = require('lowdb/adapters/FileSync') +const FileSync = require('lowdb/adapters/FileSync'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -56,43 +57,18 @@ const admin_token = '4241b401-7236-493e-92b5-b72696b9d853'; // logging setup -// console format -const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => { - return `${timestamp} ${level.toUpperCase()}: ${message}`; -}); -const logger = winston.createLogger({ - level: 'info', - format: winston.format.combine(winston.format.timestamp(), defaultFormat), - defaultMeta: {}, - transports: [ - // - // - Write to all logs with level `info` and below to `combined.log` - // - Write all logs error (and below) to `error.log`. - // - new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }), - new winston.transports.File({ filename: 'appdata/logs/combined.log' }), - new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'}) - ] -}); - -config_api.initialize(logger); -auth_api.initialize(users_db, logger); -db_api.initialize(db, users_db, logger); -subscriptions_api.initialize(db, users_db, logger, db_api); - -// var GithubContent = require('github-content'); +config_api.initialize(); +db_api.initialize(db, users_db); +auth_api.initialize(db_api); +downloader_api.initialize(db_api); +subscriptions_api.initialize(db_api, downloader_api); +categories_api.initialize(db_api); // Set some defaults db.defaults( { - playlists: { - audio: [], - video: [] - }, - files: { - audio: [], - video: [] - }, + playlists: [], + files: [], configWriteFlag: false, downloads: {}, subscriptions: [], @@ -124,39 +100,35 @@ users_db.defaults( ).write(); // config values -var frontendUrl = null; -var backendUrl = null; -var backendPort = null; -var basePath = null; -var audioFolderPath = null; -var videoFolderPath = null; -var downloadOnlyMode = null; -var useDefaultDownloadingAgent = null; -var customDownloadingAgent = null; -var allowSubscriptions = null; -var subscriptionsCheckInterval = null; -var archivePath = path.join(__dirname, 'appdata', 'archives'); +let url = null; +let backendPort = null; +let useDefaultDownloadingAgent = null; +let customDownloadingAgent = null; +let allowSubscriptions = null; +let archivePath = path.join(__dirname, 'appdata', 'archives'); // other needed values -var url_domain = null; -var updaterStatus = null; +let url_domain = null; +let updaterStatus = null; -var timestamp_server_start = Date.now(); +const concurrentStreams = {}; if (debugMode) logger.info('YTDL-Material in debug mode!'); // check if just updated -const just_restarted = fs.existsSync('restart.json'); -if (just_restarted) { +const just_updated = fs.existsSync('restart_update.json'); +if (just_updated) { updaterStatus = { updating: false, details: 'Update complete! You are now on ' + CONSTS['CURRENT_VERSION'] } - fs.unlinkSync('restart.json'); + fs.unlinkSync('restart_update.json'); } -// updates & starts youtubedl -startYoutubeDL(); +if (fs.existsSync('restart_general.json')) fs.unlinkSync('restart_general.json'); + +// updates & starts youtubedl (commented out b/c of repo takedown) +// startYoutubeDL(); var validDownloadingAgents = [ 'aria2c', @@ -170,10 +142,17 @@ var validDownloadingAgents = [ const subscription_timeouts = {}; +let version_info = null; +if (fs.existsSync('version.json')) { + version_info = fs.readJSONSync('version.json'); + logger.verbose(`Version info: ${JSON.stringify(version_info, null, 2)}`); +} else { + version_info = {'type': 'N/A', 'tag': 'N/A', 'commit': 'N/A', 'date': 'N/A'}; +} + // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); let writeConfigMode = process.env.write_ytdl_config; -var config = null; // checks if config exists, if not, a config is auto generated config_api.configExistsCheck(); @@ -184,8 +163,6 @@ if (writeConfigMode) { loadConfig(); } -var downloads = {}; - app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); @@ -194,60 +171,72 @@ app.use(auth_api.passport.initialize()); // actual functions -/** - * setTimeout, but its a promise. - * @param {number} ms - */ -async function wait(ms) { - await new Promise(resolve => { - setTimeout(resolve, ms); - }); -} - async function checkMigrations() { - // 3.5->3.6 migration - const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); + // 4.1->4.2 migration + + const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value(); + if (!simplified_db_migration_complete) { + logger.info('Beginning migration: 4.1->4.2+') + let success = await simplifyDBFileStructure(); + success = success && await db_api.addMetadataPropertyToDB('view_count'); + success = success && await db_api.addMetadataPropertyToDB('description'); + success = success && await db_api.addMetadataPropertyToDB('height'); + success = success && await db_api.addMetadataPropertyToDB('abr'); + // sets migration to complete + db.set('simplified_db_migration_complete', true).write(); + if (success) { logger.info('4.1->4.2+ migration complete!'); } + else { logger.error('Migration failed: 4.1->4.2+'); } + } - if (!files_to_db_migration_complete) { - logger.info('Beginning migration: 3.5->3.6+') - const success = await runFilesToDBMigration() - if (success) { logger.info('3.5->3.6+ migration complete!'); } - else { logger.error('Migration failed: 3.5->3.6+'); } + const new_db_system_migration_complete = db.get('new_db_system_migration_complete').value(); + if (!new_db_system_migration_complete) { + logger.info('Beginning migration: 4.2->4.3+') + let success = await db_api.importJSONToDB(db.value(), users_db.value()); + + // sets migration to complete + db.set('new_db_system_migration_complete', true).write(); + if (success) { logger.info('4.2->4.3+ migration complete!'); } + else { logger.error('Migration failed: 4.2->4.3+'); } } return true; } -async function runFilesToDBMigration() { - try { - let mp3s = await getMp3s(); - let mp4s = await getMp4s(); +async function simplifyDBFileStructure() { + // back up db files + const old_db_file = fs.readJSONSync('./appdata/db.json'); + const old_users_db_file = fs.readJSONSync('./appdata/users.json'); + fs.writeJSONSync('appdata/db.old.json', old_db_file); + fs.writeJSONSync('appdata/users.old.json', old_users_db_file); - for (let i = 0; i < mp3s.length; i++) { - let file_obj = mp3s[i]; - const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); - } + // simplify + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + const user = users[i]; + if (user['files']['video'] !== undefined && user['files']['audio'] !== undefined) { + const user_files = user['files']['video'].concat(user['files']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({files: user_files}).write(); } - - for (let i = 0; i < mp4s.length; i++) { - let file_obj = mp4s[i]; - const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); - if (!file_already_in_db) { - logger.verbose(`Migrating file ${file_obj.id}`); - await db_api.registerFileDB(file_obj.id + '.mp4', 'video'); - } + if (user['playlists']['video'] !== undefined && user['playlists']['audio'] !== undefined) { + const user_playlists = user['playlists']['video'].concat(user['playlists']['audio']); + const user_db_path = users_db.get('users').find({uid: user['uid']}); + user_db_path.assign({playlists: user_playlists}).write(); } - - // sets migration to complete - db.set('files_to_db_migration_complete', true).write(); - return true; - } catch(err) { - logger.error(err); - return false; } + + if (db.get('files.video').value() !== undefined && db.get('files.audio').value() !== undefined) { + const files = db.get('files.video').value().concat(db.get('files.audio').value()); + db.assign({files: files}).write(); + } + + if (db.get('playlists.video').value() !== undefined && db.get('playlists.audio').value() !== undefined) { + const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio').value()); + db.assign({playlists: playlists}).write(); + } + + + return true; } async function startServer() { @@ -264,18 +253,12 @@ async function startServer() { }); } -async function restartServer() { - const restartProcess = () => { - spawn('node', ['app.js'], { - detached: true, - stdio: 'inherit' - }).unref() - process.exit() - } - logger.info('Update complete! Restarting server...'); +async function restartServer(is_update = false) { + logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`); // the following line restarts the server through nodemon - fs.writeFileSync('restart.json', 'internal use only'); + fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only'); + process.exit(1); } async function updateServer(tag) { @@ -318,8 +301,9 @@ async function updateServer(tag) { updating: true, 'details': 'Update complete! Restarting server...' } - restartServer(); + restartServer(true); }, err => { + logger.error(err); updaterStatus = { updating: false, error: true, @@ -350,12 +334,10 @@ async function downloadReleaseFiles(tag) { fs.createReadStream(path.join(__dirname, `youtubedl-material-release-${tag}.zip`)).pipe(unzipper.Parse()) .on('entry', function (entry) { var fileName = entry.path; - var type = entry.type; // 'Directory' or 'File' - var size = entry.size; var is_dir = fileName.substring(fileName.length-1, fileName.length) === '/' if (!is_dir && fileName.includes('youtubedl-material/public/')) { // get public folder files - var actualFileName = fileName.replace('youtubedl-material/public/', ''); + const actualFileName = fileName.replace('youtubedl-material/public/', ''); if (actualFileName.length !== 0 && actualFileName.substring(actualFileName.length-1, actualFileName.length) !== '/') { fs.ensureDirSync(path.join(__dirname, 'public', path.dirname(actualFileName))); entry.pipe(fs.createWriteStream(path.join(__dirname, 'public', actualFileName))); @@ -364,7 +346,7 @@ async function downloadReleaseFiles(tag) { } } else if (!is_dir && !replace_ignore_list.includes(fileName)) { // get package.json - var actualFileName = fileName.replace('youtubedl-material/', ''); + const actualFileName = fileName.replace('youtubedl-material/', ''); logger.verbose('Downloading file ' + actualFileName); entry.pipe(fs.createWriteStream(path.join(__dirname, actualFileName))); } else { @@ -463,7 +445,7 @@ async function backupServerLite() { }); // wait a tiny bit for the zip to reload in fs - await wait(100); + await utils.wait(100); return true; } @@ -493,9 +475,10 @@ async function getLatestVersion() { async function killAllDownloads() { const lookupAsync = promisify(ps.lookup); + let resultList = null; try { - await lookupAsync({ + resultList = await lookupAsync({ command: 'youtube-dl' }); } catch (err) { @@ -531,7 +514,7 @@ async function killAllDownloads() { async function setPortItemFromENV() { config_api.setConfigItem('ytdl_port', backendPort.toString()); - await wait(100); + await utils.wait(100); return true; } @@ -545,7 +528,7 @@ async function setConfigFromEnv() { let success = config_api.setConfigItems(config_items); if (success) { logger.info('Config items set using ENV variables.'); - await wait(100); + await utils.wait(100); return true; } else { logger.error('ERROR: Failed to set config items using ENV variables.'); @@ -556,25 +539,36 @@ async function setConfigFromEnv() { async function loadConfig() { loadConfigValues(); + // connect to DB + await db_api.connectToDB(); + db_api.database_initialized = true; + db_api.database_initialized_bs.next(true); + // creates archive path if missing await fs.ensureDir(archivePath); - // get subscriptions - if (allowSubscriptions) { - // runs initially, then runs every ${subscriptionCheckInterval} seconds - watchSubscriptions(); - setInterval(() => { - watchSubscriptions(); - }, subscriptionsCheckInterval * 1000); - } - - db_api.importUnregisteredFiles(); - // check migrations await checkMigrations(); - // load in previous downloads - downloads = db.get('downloads').value(); + // now this is done here due to youtube-dl's repo takedown + await startYoutubeDL(); + + // get subscriptions + if (allowSubscriptions) { + // set downloading to false + let subscriptions = await subscriptions_api.getAllSubscriptions(); + subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); + // runs initially, then runs every ${subscriptionCheckInterval} seconds + const watchSubscriptionsInterval = function() { + watchSubscriptions(); + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); + setTimeout(watchSubscriptionsInterval, subscriptionsCheckInterval*1000); + } + + watchSubscriptionsInterval(); + } + + db_api.importUnregisteredFiles(); // start the server here startServer(); @@ -585,13 +579,9 @@ async function loadConfig() { function loadConfigValues() { url = !debugMode ? config_api.getConfigItem('ytdl_url') : 'http://localhost:4200'; backendPort = config_api.getConfigItem('ytdl_port'); - audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); - videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode'); useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent'); customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent'); allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions'); - subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { logger.info(`Using non-default downloading agent \'${customDownloadingAgent}\'`) @@ -616,33 +606,27 @@ function loadConfigValues() { function calculateSubcriptionRetrievalDelay(subscriptions_amount) { // frequency is once every 5 mins by default + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); let interval_in_ms = subscriptionsCheckInterval * 1000; const subinterval_in_ms = interval_in_ms/subscriptions_amount; return subinterval_in_ms; } async function watchSubscriptions() { - let subscriptions = null; - - const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); - if (multiUserMode) { - subscriptions = []; - let users = users_db.get('users').value(); - for (let i = 0; i < users.length; i++) { - if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); - } - } else { - subscriptions = subscriptions_api.getAllSubscriptions(); - } + let subscriptions = await subscriptions_api.getAllSubscriptions(); if (!subscriptions) return; - let subscriptions_amount = subscriptions.length; + const valid_subscriptions = subscriptions.filter(sub => !sub.paused); + + let subscriptions_amount = valid_subscriptions.length; let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount); let current_delay = 0; - for (let i = 0; i < subscriptions.length; i++) { - let sub = subscriptions[i]; + + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + for (let i = 0; i < valid_subscriptions.length; i++) { + let sub = valid_subscriptions[i]; // don't check the sub if the last check for the same subscription has not completed if (subscription_timeouts[sub.id]) { @@ -667,6 +651,7 @@ async function watchSubscriptions() { }, current_delay); subscription_timeouts[sub.id] = true; current_delay += delay_interval; + const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; } } @@ -696,829 +681,17 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -async function getMp3s() { - let mp3s = []; - var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(audioFolderPath.length, file.length); - - var stats = await fs.stat(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp3(id, audioFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = true; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp3s.push(file_obj); - } - return mp3s; -} - -async function getMp4s(relative_path = true) { - let mp4s = []; - var files = await utils.recFindByExt(videoFolderPath, 'mp4'); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(videoFolderPath.length, file.length); - - var stats = fs.statSync(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp4(id, videoFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - - var size = stats.size; - - var isaudio = false; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp4s.push(file_obj); - } - return mp4s; -} - -function getThumbnailMp3(name) -{ - var obj = utils.getJSONMp3(name, audioFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getThumbnailMp4(name) -{ - var obj = utils.getJSONMp4(name, videoFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getFileSizeMp3(name) -{ - var jsonPath = audioFolderPath+name+".mp3.info.json"; - - if (fs.existsSync(jsonPath)) - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - else - var obj = 0; - - return obj.filesize; -} - -function getFileSizeMp4(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - var filesize = 0; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - for (i = 0; i < obj.formats.length; i++) - { - if (obj.formats[i].format_id == format) - { - filesize = obj.formats[i].filesize; - } - } - } - - return filesize; -} - -function getAmountDownloadedMp3(name) -{ - var partPath = audioFolderPath+name+".mp3.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - - - -function getAmountDownloadedMp4(name) -{ - var format = getVideoFormatID(name); - var partPath = videoFolderPath+name+".f"+format+".mp4.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - -function getVideoFormatID(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - return format; - } -} - -async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { - let zipFolderPath = null; - - if (!fullPathProvided) { - zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath); - if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); - } else { - zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - - let ext = (type === 'audio') ? '.mp3' : '.mp4'; - - let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip')); - - var archive = archiver('zip', { - gzip: true, - zlib: { level: 9 } // Sets the compression level. - }); - - archive.on('error', function(err) { - logger.error(err); - throw err; - }); - - // pipe archive data to the output file - archive.pipe(output); - - for (let i = 0; i < fileNames.length; i++) { - let fileName = fileNames[i]; - let fileNamePathRemoved = path.parse(fileName).base; - let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName; - archive.file(file_path, {name: fileNamePathRemoved + ext}) - } - - await archive.finalize(); - - // wait a tiny bit for the zip to reload in fs - await wait(100); - return path.join(zipFolderPath,outputName + '.zip'); -} - -async function deleteAudioFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : audioFolderPath; - - var jsonPath = path.join(filePath,name+'.mp3.info.json'); - var altJSONPath = path.join(filePath,name+'.info.json'); - var audioFilePath = path.join(filePath,name+'.mp3'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - altJSONPath = path.join(__dirname, altJSONPath); - audioFilePath = path.join(__dirname, audioFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - let audioFileExists = await fs.pathExists(audioFilePath); - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_audio.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp3(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('audio', line); - } else { - logger.info('Could not find archive file for audio files. Creating...'); - await fs.close(await fs.open(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (audioFileExists) { - await fs.unlink(audioFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(audioFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -async function deleteVideoFile(name, customPath = null, blacklistMode = false) { - let filePath = customPath ? customPath : videoFolderPath; - var jsonPath = path.join(filePath,name+'.info.json'); - - var altJSONPath = path.join(filePath,name+'.mp4.info.json'); - var videoFilePath = path.join(filePath,name+'.mp4'); - var thumbnailPath = path.join(filePath,name+'.webp'); - var altThumbnailPath = path.join(filePath,name+'.jpg'); - - jsonPath = path.join(__dirname, jsonPath); - videoFilePath = path.join(__dirname, videoFilePath); - - let jsonExists = await fs.pathExists(jsonPath); - let videoFileExists = await fs.pathExists(videoFilePath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - if (config_api.descriptors[name]) { - try { - for (let i = 0; i < config_api.descriptors[name].length; i++) { - config_api.descriptors[name][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); - - // get ID from JSON - - var jsonobj = await utils.getJSONMp4(name, filePath); - let id = null; - if (jsonobj) id = jsonobj.id; - - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist('video', line); - } else { - logger.info('Could not find archive file for videos. Creating...'); - fs.closeSync(fs.openSync(archive_path, 'w')); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - if (videoFileExists) { - await fs.unlink(videoFilePath); - if (await fs.pathExists(jsonPath) || await fs.pathExists(videoFilePath)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -/** - * @param {'audio' | 'video'} type - * @param {string[]} fileNames - */ -async function getAudioOrVideoInfos(type, fileNames) { - let result = await Promise.all(fileNames.map(async fileName => { - let fileLocation = videoFolderPath+fileName; - if (type === 'audio') { - fileLocation += '.mp3.info.json'; - } else if (type === 'video') { - fileLocation += '.info.json'; - } - - if (await fs.pathExists(fileLocation)) { - let data = await fs.readFile(fileLocation); - try { - return JSON.parse(data); - } catch (e) { - let suffix; - if (type === 'audio') { - suffix += '.mp3'; - } else if (type === 'video') { - suffix += '.mp4'; - } - - logger.error(`Could not find info for file ${fileName}${suffix}`); - } - } - return null; - })); - - return result.filter(data => data != null); -} - -// downloads - -async function downloadFileByURL_exec(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - - // audio / video specific vars - var is_audio = type === 'audio'; - var ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'exec'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now(), - filesize: null - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - // get video info prior to download - const info = await getVideoInfoByURL(url, downloadConfig, download); - if (!info) { - resolve(false); - return; - } else { - // store info in download for future use - download['_filename'] = info['_filename']; - download['filesize'] = utils.getExpectedFileSize(info); - } - - const download_checker = setInterval(() => checkDownloadPercent(download), 1000); - - // download file - youtubedl.exec(url, downloadConfig, {}, function(err, output) { - clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) - - download['downloading'] = false; - download['timestamp_end'] = Date.now(); - var file_uid = null; - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); - if (err) { - logger.error(err.stderr); - - download['error'] = err.stderr; - updateDownloads(); - resolve(false); - return; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { - download['error'] = 'No output. Check if video already exists in your archive.'; - logger.warn(`No output received for video download, check if it exists in your archive.`) - updateDownloads(); - - resolve(false); - return; - } - var file_names = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - var modified_file_name = output_json ? output_json['title'] : null; - if (!output_json) { - continue; - } - - // get filepath with no extension - const filepath_no_extension = removeFileExtension(output_json['_filename']); - - var full_file_path = filepath_no_extension + ext; - var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); - - // renames file if necessary due to bug - if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { - try { - fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); - logger.info('Renamed ' + file_name + '.webm to ' + file_name); - } catch(e) { - } - } - - if (type === 'audio') { - let tags = { - title: output_json['title'], - artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] - } - let success = NodeID3.write(tags, output_json['_filename']); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); - } - - // registers file in DB - file_uid = db_api.registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode); - - if (file_name) file_names.push(file_name); - } - - let is_playlist = file_names.length > 1; - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - download['complete'] = true; - download['fileNames'] = is_playlist ? file_names : [full_file_path] - updateDownloads(); - - var videopathEncoded = encodeURIComponent(file_names[0]); - - resolve({ - [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: is_playlist ? file_names : null, - uid: file_uid - }); - } - }); - }); -} - -async function downloadFileByURL_normal(url, type, options, sessionID = null) { - return new Promise(async resolve => { - var date = Date.now(); - var file_uid = null; - const is_audio = type === 'audio'; - const ext = is_audio ? '.mp3' : '.mp4'; - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; } - - // prepend with user if needed - let multiUserMode = null; - if (options.user) { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_path = path.join(usersFileFolder, options.user, type); - fs.ensureDirSync(user_path); - fileFolderPath = user_path + path.sep; - multiUserMode = { - user: options.user, - file_path: fileFolderPath - } - options.customFileFolderPath = fileFolderPath; - } - - options.downloading_method = 'normal'; - const downloadConfig = await generateArgs(url, type, options); - - // adds download to download helper - const download_uid = uuid(); - const session = sessionID ? sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - ui_uid: options.ui_uid, - downloading: true, - complete: false, - url: url, - type: type, - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - const download = downloads[session][download_uid]; - updateDownloads(); - - const video = youtubedl(url, - // Optional arguments passed to youtube-dl. - downloadConfig, - // Additional options can be given for calling `child_process.execFile()`. - { cwd: __dirname }); - - let video_info = null; - let file_size = 0; - - // Will be called when the download starts. - video.on('info', function(info) { - video_info = info; - file_size = video_info.size; - const json_path = removeFileExtension(video_info._filename) + '.info.json'; - fs.ensureFileSync(json_path); - fs.writeJSONSync(json_path, video_info); - video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) - }); - // Will be called if download was already completed and there is nothing more to download. - video.on('complete', function complete(info) { - 'use strict' - logger.info('file ' + info._filename + ' already downloaded.') - }) - - let download_pos = 0; - video.on('data', function data(chunk) { - download_pos += chunk.length - // `size` should not be 0 here. - if (file_size) { - let percent = (download_pos / file_size * 100).toFixed(2) - download['percent_complete'] = percent; - } - }); - - video.on('end', async function() { - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - download['timestamp_end'] = Date.now(); - download['fileNames'] = [removeFileExtension(video_info._filename) + ext]; - download['complete'] = true; - updateDownloads(); - - // audio-only cleanup - if (is_audio) { - // filename fix - video_info['_filename'] = removeFileExtension(video_info['_filename']) + '.mp3'; - - // ID3 tagging - let tags = { - title: video_info['title'], - artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] - } - let success = NodeID3.write(tags, video_info._filename); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); - - const possible_webm_path = removeFileExtension(video_info['_filename']) + '.webm'; - const possible_mp4_path = removeFileExtension(video_info['_filename']) + '.mp4'; - // check if audio file is webm - if (fs.existsSync(possible_webm_path)) await convertFileToMp3(possible_webm_path, video_info['_filename']); - else if (fs.existsSync(possible_mp4_path)) await convertFileToMp3(possible_mp4_path, video_info['_filename']); - } - - // registers file in DB - const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); - - if (options.merged_string !== null && options.merged_string !== undefined) { - let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); - let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - - videopathEncoded = encodeURIComponent(removeFileExtension(base_file_name)); - - resolve({ - [is_audio ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, - file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready - uid: file_uid - }); - }); - - video.on('error', function error(err) { - logger.error(err); - - download[error] = err; - updateDownloads(); - - resolve(false); - }); - }); - -} - -async function generateArgs(url, type, options) { - var videopath = '%(title)s'; - var globalArgs = config_api.getConfigItem('ytdl_custom_args'); - let useCookies = config_api.getConfigItem('ytdl_use_cookies'); - var is_audio = type === 'audio'; - - var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; - - if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath; - - var customArgs = options.customArgs; - var customOutput = options.customOutput; - var customQualityConfiguration = options.customQualityConfiguration; - - // video-specific args - var selectedHeight = options.selectedHeight; - - // audio-specific args - var maxBitrate = options.maxBitrate; - - var youtubeUsername = options.youtubeUsername; - var youtubePassword = options.youtubePassword; - - let downloadConfig = null; - let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4']; - const is_youtube = url.includes('youtu'); - if (!is_audio && !is_youtube) { - // tiktok videos fail when using the default format - qualityPath = null; - } else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) { - qualityPath = ['-f', 'bestvideo+bestaudio'] - } - - if (customArgs) { - downloadConfig = customArgs.split(',,'); - } else { - if (customQualityConfiguration) { - qualityPath = ['-f', customQualityConfiguration]; - } else if (selectedHeight && selectedHeight !== '' && !is_audio) { - qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`]; - } else if (maxBitrate && is_audio) { - qualityPath = ['--audio-quality', maxBitrate] - } - - if (customOutput) { - downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json']; - } else { - downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; - } - - if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath); - - if (is_audio && !options.skip_audio_args) { - downloadConfig.push('-x'); - downloadConfig.push('--audio-format', 'mp3'); - } - - if (youtubeUsername && youtubePassword) { - downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); - } - - if (useCookies) { - if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) { - downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); - } else { - logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); - } - } - - if (!useDefaultDownloadingAgent && customDownloadingAgent) { - downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath; - const archive_path = path.join(archive_folder, `archive_${type}.txt`); - - await fs.ensureDir(archive_folder); - - // create archive file if it doesn't exist - if (!(await fs.pathExists(archive_path))) { - await fs.close(await fs.open(archive_path, 'w')); - } - - let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); - // create blacklist file if it doesn't exist - if (!(await fs.pathExists(blacklist_path))) { - await fs.close(await fs.open(blacklist_path, 'w')); - } - - let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); - await fs.ensureFile(merged_path); - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - let status = await mergeFiles(inputPathList, merged_path); - - options.merged_string = await fs.readFile(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - - if (config_api.getConfigItem('ytdl_include_thumbnail')) { - downloadConfig.push('--write-thumbnail'); - } - - if (globalArgs && globalArgs !== '') { - // adds global args - if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) { - // if global args has an output, replce the original output with that of global args - const original_output_index = downloadConfig.indexOf('-o'); - downloadConfig.splice(original_output_index, 2); - } - downloadConfig = downloadConfig.concat(globalArgs.split(',,')); - } - - } - logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); - return downloadConfig; -} - -async function getVideoInfoByURL(url, args = [], download = null) { - return new Promise(resolve => { - // remove bad args - const new_args = [...args]; - - const archiveArgIndex = new_args.indexOf('--download-archive'); - if (archiveArgIndex !== -1) { - new_args.splice(archiveArgIndex, 2); - } - - // actually get info - youtubedl.getInfo(url, new_args, (err, output) => { - if (output) { - resolve(output); - } else { - logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`); - if (download) { - download['error'] = `Failed pre-check for video info: ${err}`; - updateDownloads(); - } - resolve(null); - } - }); - }); -} - // currently only works for single urls -async function getUrlInfos(urls) { +async function getUrlInfos(url) { let startDate = Date.now(); let result = []; return new Promise(resolve => { - youtubedl.exec(urls.join(' '), ['--dump-json'], {}, (err, output) => { + youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { let new_date = Date.now(); let difference = (new_date - startDate)/1000; logger.debug(`URL info retrieval delay: ${difference} seconds.`); if (err) { - logger.error('Error during parsing:' + err); + logger.error(`Error during retrieving formats for ${url}: ${err}`); resolve(null); } let try_putput = null; @@ -1526,97 +699,51 @@ async function getUrlInfos(urls) { try_putput = JSON.parse(output); result = try_putput; } catch(e) { - // probably multiple urls - logger.error('failed to parse for urls starting with ' + urls[0]); - // logger.info(output); + logger.error(`Failed to retrieve available formats for url: ${url}`); } resolve(result); }); }); } -async function convertFileToMp3(input_file, output_file) { - logger.verbose(`Converting ${input_file} to ${output_file}...`); - return new Promise(resolve => { - ffmpeg(input_file).noVideo().toFormat('mp3') - .on('end', () => { - logger.verbose(`Conversion for '${output_file}' complete.`); - fs.unlinkSync(input_file) - resolve(true); - }) - .on('error', (err) => { - logger.error('Failed to convert audio file to the correct format.'); - logger.error(err); - resolve(false); - }).save(output_file); - }); -} - -async function writeToBlacklist(type, line) { - let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); - // adds newline to the beginning of the line - line = '\n' + line; - await fs.appendFile(blacklistPath, line); -} - -// download management functions - -function updateDownloads() { - db.assign({downloads: downloads}).write(); -} - -function checkDownloadPercent(download) { - /* - This is more of an art than a science, we're just selecting files that start with the file name, - thus capturing the parts being downloaded in files named like so: '