TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into api-generator

Browse Source
This commit is contained in:
Isaac Abadi
2021-09-29 02:38:42 -06:00
parent 2cf0c61fac f0c3837ee5
commit 3937700eff
199 changed files with 46018 additions and 15941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
browserslist → .browserslistrc
View File

20
.eslintrc.json Normal file
View File

@@ -0,0 +1,20 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
}

31
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,31 @@
---
name: Bug report
about: Create a report to help us improve
title: "[BUG]"
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Environment**
- YoutubeDL-Material version
- Docker tag: <tag> (optional)
**Additional context**
Add any other context about the problem here. For example, a YouTube link.

17
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@@ -0,0 +1,17 @@
---
name: Feature request
about: Suggest an idea for this project
title: "[FEATURE]"
labels: enhancement
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Additional context**
Add any other context or screenshots about the feature request here.

111
.github/workflows/build.yml vendored Normal file
View File

@@ -0,0 +1,111 @@
name: continuous integration
on:
push:
branches: [master, feat/*]
tags:
- v*
pull_request:
branches: [master]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: setup node
uses: actions/setup-node@v2
with:
node-version: '12'
cache: 'npm'
- name: install dependencies
run: |
npm install
cd backend
npm install
sudo npm install -g @angular/cli
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: build
run: npm run build
- name: prepare artifact upload
shell: pwsh
run: |
New-Item -Name build -ItemType Directory
New-Item -Path build -Name youtubedl-material -ItemType Directory
Copy-Item -Path ./backend/appdata -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/audio -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/authentication -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
- name: upload build artifact
uses: actions/upload-artifact@v1
with:
name: youtubedl-material
path: build
release:
runs-on: ubuntu-latest
needs: build
if: contains(github.ref, '/tags/v')
steps:
- name: checkout code
uses: actions/checkout@v2
- name: create release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: YoutubeDL-Material ${{ github.ref }}
body: |
# New features
# Minor additions
# Bug fixes
draft: true
prerelease: false
- name: download build artifact
uses: actions/download-artifact@v1
with:
name: youtubedl-material
path: ${{runner.temp}}/youtubedl-material
- name: extract tag name
id: tag_name
run: echo ::set-output name=tag_name::${GITHUB_REF#refs/tags/}
- name: prepare release asset
shell: pwsh
run: Compress-Archive -Path ${{runner.temp}}/youtubedl-material -DestinationPath youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
- name: upload release asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
asset_name: youtubedl-material-${{ steps.tag_name.outputs.tag_name }}.zip
asset_content_type: application/zip
- name: upload docker-compose asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./docker-compose.yml
asset_name: docker-compose.yml
asset_content_type: text/plain

45
.github/workflows/docker-release.yml vendored Normal file
View File

@@ -0,0 +1,45 @@
name: docker-release
on:
workflow_dispatch:
inputs:
tags:
description: 'Docker tags'
required: true
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: build & push images
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: ${{ github.event.inputs.tags }}

42
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,42 @@
name: docker
on:
push:
branches: [master]
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: build & push images
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: tzahi12345/youtubedl-material:nightly

1
.gitignore vendored
View File

@@ -66,3 +66,4 @@ backend/appdata/users.json
backend/users/*
backend/appdata/cookies.txt
backend/public
src/assets/i18n/*.json

25
.vscode/tasks.json vendored Normal file
View File

@@ -0,0 +1,25 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "npm",
"script": "start",
"problemMatcher": [],
"label": "Dev: start frontend",
"detail": "ng serve"
},
{
"label": "Dev: start backend",
"type": "shell",
"command": "set YTDL_MODE=debug && node app.js",
"options": {
"cwd": "./backend"
},
"presentation": {
"reveal": "always",
"panel": "new"
},
"problemMatcher": []
}
]
}

12
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM alpine:3.12 as frontend
FROM alpine:latest as frontend
RUN apk add --no-cache \
npm
@@ -11,28 +11,32 @@ RUN npm install
COPY [ "angular.json", "tsconfig.json", "/build/" ]
COPY [ "src/", "/build/src/" ]
RUN ng build --prod
RUN npm run build
#--------------#
FROM alpine:3.12
FROM alpine:latest
ENV UID=1000 \
GID=1000 \
USER=youtube
ENV NO_UPDATE_NOTIFIER=true
RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
RUN apk add --no-cache \
ffmpeg \
npm \
python2 \
python3 \
su-exec \
&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
atomicparsley
WORKDIR /app
COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
RUN npm install pm2 -g
RUN npm install && chown -R $UID:$GID ./
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
@@ -40,4 +44,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "node", "app.js" ]
CMD [ "pm2-runtime", "pm2.config.js" ]

4
Public API v1.yaml
View File

@@ -282,12 +282,12 @@ paths:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/getAllSubscriptions:
/api/getSubscriptions:
post:
tags:
- subscriptions
summary: Get all subscriptions
operationId: post-api-getAllSubscriptions
operationId: post-api-getSubscriptions
requestBody:
content:
application/json:

53
README.md
View File

@@ -1,12 +1,12 @@
# YoutubeDL-Material
[![](https://img.shields.io/docker/pulls/tzahi12345/youtubedl-material.svg)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
[![](https://img.shields.io/docker/image-size/tzahi12345/youtubedl-material?sort=date)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
[![](https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg)](https://heroku.com/deploy?template=https://github.com/Tzahi12345/YoutubeDL-Material)
[![](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
[![](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
[![Docker pulls badge](https://img.shields.io/docker/pulls/tzahi12345/youtubedl-material.svg)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
[![Docker image size badge](https://img.shields.io/docker/image-size/tzahi12345/youtubedl-material?sort=date)](https://hub.docker.com/r/tzahi12345/youtubedl-material)
[![Heroku deploy badge](https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg)](https://heroku.com/deploy?template=https://github.com/Tzahi12345/YoutubeDL-Material)
[![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
[![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 9](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
Now with [Docker](#Docker) support!
@@ -16,27 +16,35 @@ Check out the prerequisites, and go to the installation section. Easy as pie!
Here's an image of what it'll look like once you're done:
![frontpage](https://i.imgur.com/w8iofbb.png)
With optional file management enabled (default):
![frontpage_with_files](https://i.imgur.com/FTATqBM.png)
<img src="https://i.imgur.com/C6vFGbL.png" width="800">
Dark mode:
![dark_mode](https://i.imgur.com/r5ZtBqd.png)
<img src="https://i.imgur.com/vOtvH5w.png" width="800">
### Prerequisites
NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
Make sure you have these dependencies installed on your system: nodejs and youtube-dl. If you don't, run this command:
Debian/Ubuntu:
```bash
sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
```
sudo apt-get install nodejs youtube-dl
CentOS 7:
```bash
sudo yum install epel-release
sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm
sudo yum install centos-release-scl-rh
sudo yum install rh-nodejs12
scl enable rh-nodejs12 bash
sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
```
Optional dependencies:
* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
### Installing
@@ -59,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
@@ -69,20 +77,26 @@ Alternatively, you can port forward the port specified in the config (defaults t
## Docker
### Host-specific instructions
If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
### Setup
If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
1. Run `curl -L https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest/download/docker-compose.yml -o docker-compose.yml` to download the latest Docker Compose, or go to the [releases](https://github.com/Tzahi12345/YoutubeDL-Material/releases/) page to grab the version you'd like.
2. Run `docker-compose pull`. This will download the official YoutubeDL-Material docker image.
3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 8998" or something similar.
4. Make sure you can connect to the specified URL + port, and if so, you are done!
3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
### Custom UID/GID
By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
```
```yml
environment:
UID: YOUR_UID
GID: YOUR_GID
@@ -109,11 +123,12 @@ If you're interested in translating the app into a new language, check out the [
* **Isaac Grynsztein** (me!) - *Initial work*
Official translators:
* Spanish - tzahi12345
* German - UnlimitedCookies
* Chinese - TyRoyal
See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
## License

2
angular.json
View File

@@ -45,8 +45,6 @@
],
"optimization": true,
"outputHashing": "all",
"sourceMap": false,
"extractCss": true,
"namedChunks": false,
"aot": true,
"extractLicenses": true,

18
backend/.eslintrc.json Normal file
View File

@@ -0,0 +1,18 @@
{
"env": {
"node": true,
"es2021": true
},
"extends": [
"eslint:recommended"
],
"parser": "esprima",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [],
"rules": {
},
"root": true
}

2412
backend/app.js
View File

File diff suppressed because it is too large Load Diff

29
backend/appdata/default.json
View File

@@ -7,25 +7,34 @@
"Downloader": {
"path-audio": "audio/",
"path-video": "video/",
"default_file_output": "",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"enable_downloads_manager": true
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": ""
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",
@@ -34,7 +43,8 @@
"Subscriptions": {
"allow_subscriptions": true,
"subscriptions_base_path": "subscriptions/",
"subscriptions_check_interval": "300"
"subscriptions_check_interval": "300",
"redownload_fresh_uploads": false
},
"Users": {
"base_path": "users/",
@@ -48,14 +58,19 @@
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": true,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,
"custom_downloading_agent": "",
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"jwt_expiration": 86400,
"jwt_expiration": 86400,
"logger_level": "info"
}
}
}
}

276
backend/authentication/auth.js
View File

@@ -1,12 +1,10 @@
const path = require('path');
const config_api = require('../config');
const consts = require('../consts');
var subscriptions_api = require('../subscriptions')
const fs = require('fs-extra');
var jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
var bcrypt = require('bcryptjs');
const logger = require('../logger');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -14,16 +12,14 @@ var JwtStrategy = require('passport-jwt').Strategy,
ExtractJwt = require('passport-jwt').ExtractJwt;
// other required vars
let logger = null;
var users_db = null;
let db_api = null;
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function(input_users_db, input_logger) {
setLogger(input_logger)
setDB(input_users_db);
exports.initialize = function(db_api) {
setDB(db_api);
/*************************
* Authentication module
@@ -33,21 +29,19 @@ exports.initialize = function(input_users_db, input_logger) {
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
SERVER_SECRET = null;
if (users_db.get('jwt_secret').value()) {
SERVER_SECRET = users_db.get('jwt_secret').value();
if (db_api.users_db.get('jwt_secret').value()) {
SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
} else {
SERVER_SECRET = uuid();
users_db.set('jwt_secret', SERVER_SECRET).write();
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
/*opts.issuer = 'example.com';
opts.audience = 'example.com';*/
exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) {
const user = users_db.get('users').find({uid: jwt_payload.user}).value();
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
} else {
@@ -57,12 +51,8 @@ exports.initialize = function(input_users_db, input_logger) {
}));
}
function setLogger(input_logger) {
logger = input_logger;
}
function setDB(input_users_db) {
users_db = input_users_db;
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport');
@@ -78,7 +68,7 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = function(req, res) {
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
@@ -89,21 +79,27 @@ exports.registerUser = function(req, res) {
return;
}
if (plaintextPassword === "") {
res.sendStatus(400);
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
return;
}
bcrypt.hash(plaintextPassword, saltRounds)
.then(function(hash) {
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (users_db.get('users').find({uid: userid}).value()) {
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (users_db.get('users').find({name: username}).value()) {
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
@@ -136,16 +132,18 @@ exports.registerUser = function(req, res) {
************************************************/
exports.login = async (username, password) => {
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); return false }
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
}
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) {
return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
}
return done(null, await exports.login(username, password));
}
));
@@ -156,17 +154,17 @@ var getLDAPConfiguration = function(req, callback) {
};
exports.passport.use(new LdapStrategy(getLDAPConfiguration,
function(user, done) {
async function(user, done) {
// check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false);
const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value();
let db_user = await db_api.getRecord('users', {uid: user_uid});
if (!db_user) {
// generate DB user
let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
db_user = new_user;
logger.verbose(`Generated new user ${user_uid} using LDAP`);
}
@@ -190,11 +188,11 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.returnAuthResponse = function(req, res) {
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
token: req.token,
permissions: exports.userPermissions(req.user.uid),
permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -207,7 +205,7 @@ exports.returnAuthResponse = function(req, res) {
* It also passes the user object to the next
* middleware through res.locals
**************************************/
exports.ensureAuthenticatedElseError = function(req, res, next) {
exports.ensureAuthenticatedElseError = (req, res, next) => {
var token = getToken(req.query);
if( token ) {
try {
@@ -225,10 +223,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
}
// change password
exports.changeUserPassword = async function(user_uid, new_pass) {
exports.changeUserPassword = async (user_uid, new_pass) => {
try {
const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
return true;
} catch (err) {
return false;
@@ -236,16 +234,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
}
// change user permissions
exports.changeUserPermissions = function(user_uid, permission, new_value) {
exports.changeUserPermissions = async (user_uid, permission, new_value) => {
try {
const user_db_obj = users_db.get('users').find({uid: user_uid});
user_db_obj.get('permissions').pull(permission).write();
user_db_obj.get('permission_overrides').pull(permission).write();
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
if (new_value === 'yes') {
user_db_obj.get('permissions').push(permission).write();
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} else if (new_value === 'no') {
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
}
return true;
} catch (err) {
@@ -255,12 +252,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
}
// change role permissions
exports.changeRolePermissions = function(role, permission, new_value) {
exports.changeRolePermissions = async (role, permission, new_value) => {
try {
const role_db_obj = users_db.get('roles').get(role);
role_db_obj.get('permissions').pull(permission).write();
await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
if (new_value === 'yes') {
role_db_obj.get('permissions').push(permission).write();
await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
}
return true;
} catch (err) {
@@ -269,30 +265,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
}
}
exports.adminExists = function() {
return !!users_db.get('users').find({uid: 'admin'}).value();
exports.adminExists = async function() {
return !!(await db_api.getRecord('users', {uid: 'admin'}));
}
// video stuff
exports.getUserVideos = function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value();
return user['files'][type];
exports.getUserVideos = async function(user_uid, type) {
const files = await db_api.getRecords('files', {user_uid: user_uid});
return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
}
exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false) {
let file = null;
if (!type) {
file = users_db.get('users').find({uid: user_uid}).get(`files.audio`).find({uid: file_uid}).value();
if (!file) {
file = users_db.get('users').find({uid: user_uid}).get(`files.video`).find({uid: file_uid}).value();
if (file) type = 'video';
} else {
type = 'audio';
}
}
if (!file && type) file = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value();
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -300,38 +285,17 @@ exports.getUserVideo = function(user_uid, file_uid, type, requireSharing = false
return file;
}
exports.addPlaylist = function(user_uid, new_playlist, type) {
users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).push(new_playlist).write();
exports.removePlaylist = async function(user_uid, playlistID) {
await db_api.removeRecord('playlist', {playlistID: playlistID});
return true;
}
exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames, type) {
users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).assign({fileNames: new_filenames});
return true;
exports.getUserPlaylists = async function(user_uid) {
return await db_api.getRecords('playlists', {user_uid: user_uid});
}
exports.removePlaylist = function(user_uid, playlistID, type) {
users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).remove({id: playlistID}).write();
return true;
}
exports.getUserPlaylists = function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value();
return user['playlists'][type];
}
exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing = false) {
let playlist = null;
if (!type) {
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.audio`).find({id: playlistID}).value();
if (!playlist) {
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.video`).find({id: playlistID}).value();
if (playlist) type = 'video';
} else {
type = 'audio';
}
}
if (!playlist) playlist = users_db.get('users').find({uid: user_uid}).get(`playlists.${type}`).find({id: playlistID}).value();
exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
let playlist = await db_api.getRecord('playlists', {id: playlistID});
// prevent unauthorized users from accessing the file info
if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -339,108 +303,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, type, requireSharing =
return playlist;
}
exports.registerUserFile = function(user_uid, file_object, type) {
users_db.get('users').find({uid: user_uid}).get(`files.${type}`)
.remove({
path: file_object['path']
}).write();
users_db.get('users').find({uid: user_uid}).get(`files.${type}`)
.push(file_object)
.write();
}
exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) {
exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value();
if (file_obj) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
// close descriptors
if (config_api.descriptors[file_obj.id]) {
try {
for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) {
config_api.descriptors[file_obj.id][i].destroy();
}
} catch(e) {
}
}
const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext);
users_db.get('users').find({uid: user_uid}).get(`files.${type}`)
.remove({
uid: file_uid
}).write();
if (await fs.pathExists(full_path)) {
// remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null;
if (await fs.pathExists(json_path)) {
youtube_id = await fs.readJSON(json_path).id;
await fs.unlink(json_path);
} else if (await fs.pathExists(alternate_json_path)) {
youtube_id = await fs.readJSON(alternate_json_path).id;
await fs.unlink(alternate_json_path);
}
await fs.unlink(full_path);
// do archive stuff
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
} else {
logger.info(`Could not find archive file for ${type} files. Creating...`);
await fs.ensureFile(archive_path);
}
}
}
success = true;
} else {
success = false;
logger.warn(`User file ${file_uid} does not exist!`);
}
is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
success = true;
return success;
}
exports.changeSharingMode = function(user_uid, file_uid, type, is_playlist, enabled) {
let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid});
if (user_db_obj.value()) {
const file_db_obj = is_playlist ? user_db_obj.get(`playlists.${type}`).find({id: file_uid}) : user_db_obj.get(`files.${type}`).find({uid: file_uid});
if (file_db_obj.value()) {
success = true;
file_db_obj.assign({sharingEnabled: enabled}).write();
}
}
exports.userHasPermission = async function(user_uid, permission) {
return success;
}
exports.userHasPermission = function(user_uid, permission) {
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return false;
}
const role_permissions = (users_db.get('roles').value())['permissions'];
const role_permissions = (await db_api.getRecords('roles'))['permissions'];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -463,16 +342,17 @@ exports.userHasPermission = function(user_uid, permission) {
}
}
exports.userPermissions = function(user_uid) {
exports.userPermissions = async function(user_uid) {
let user_permissions = [];
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return null;
}
const role_permissions = users_db.get('roles').get(role).get('permissions').value()
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];
@@ -518,14 +398,8 @@ function generateUserObject(userid, username, hash, auth_method = 'internal') {
name: username,
uid: userid,
passhash: auth_method === 'internal' ? hash : null,
files: {
audio: [],
video: []
},
playlists: {
audio: [],
video: []
},
files: [],
playlists: [],
subscriptions: [],
created: Date.now(),
role: userid === 'admin' && auth_method === 'internal' ? 'admin' : 'user',

144
backend/categories.js Normal file
View File

@@ -0,0 +1,144 @@
const utils = require('./utils');
const logger = require('./logger');
var db_api = null;
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db_api) {
setDB(input_db_api);
}
/*
Categories:
Categories are a way to organize videos based on dynamic rules set by the user. Categories are universal (so not per-user).
Categories, besides rules, have an optional custom output. This custom output can help users create their
desired directory structure.
Rules:
A category rule consists of a property, a comparison, and a value. For example, "uploader includes 'VEVO'"
Rules are stored as an object with the above fields. In addition to those fields, it also has a preceding_operator, which
is either OR or AND, and signifies whether the rule should be ANDed with the previous rules, or just ORed. For the first
rule, this field is null.
Ex. (title includes 'Rihanna' OR title includes 'Beyonce' AND uploader includes 'VEVO')
*/
async function categorize(file_jsons) {
// to make the logic easier, let's assume the file metadata is an array
if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
let selected_category = null;
const categories = await getCategories();
if (!categories) {
logger.warn('Categories could not be found.');
return null;
}
for (let i = 0; i < file_jsons.length; i++) {
const file_json = file_jsons[i];
for (let j = 0; j < categories.length; j++) {
const category = categories[j];
const rules = category['rules'];
// if rules for current category apply, then that is the selected category
if (applyCategoryRules(file_json, rules, category['name'])) {
selected_category = category;
logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`);
return selected_category;
}
}
}
return selected_category;
}
async function getCategories() {
const categories = await db_api.getRecords('categories');
return categories ? categories : null;
}
async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = await getCategories();
if (available_categories && files) {
for (let category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
category['thumbnailPath'] = files_that_match[0].thumbnailPath;
category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
category['id'] = category['uid'];
categories_as_playlists.push(category);
}
}
}
return categories_as_playlists;
}
function applyCategoryRules(file_json, rules, category_name) {
let rules_apply = false;
for (let i = 0; i < rules.length; i++) {
const rule = rules[i];
let rule_applies = null;
let preceding_operator = rule['preceding_operator'];
switch (rule['comparator']) {
case 'includes':
rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
break;
case 'not_includes':
rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
break;
case 'equals':
rule_applies = file_json[rule['property']] === rule['value'];
break;
case 'not_equals':
rule_applies = file_json[rule['property']] !== rule['value'];
break;
default:
logger.warn(`Invalid comparison used for category ${category_name}`)
break;
}
// OR the first rule with rules_apply, which will be initially false
if (i === 0) preceding_operator = 'or';
// update rules_apply based on current rule
if (preceding_operator === 'or')
rules_apply = rules_apply || rule_applies;
else
rules_apply = rules_apply && rule_applies;
}
return rules_apply;
}
// async function addTagToVideo(tag, video, user_uid) {
// // TODO: Implement
// }
// async function removeTagFromVideo(tag, video, user_uid) {
// // TODO: Implement
// }
// // adds tag to list of existing tags (used for tag suggestions)
// async function addTagToExistingTags(tag) {
// const existing_tags = db.get('tags').value();
// if (!existing_tags.includes(tag)) {
// db.get('tags').push(tag).write();
// }
// }
module.exports = {
initialize: initialize,
categorize: categorize,
getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists
}

39
backend/config.js
View File

@@ -1,3 +1,5 @@
const logger = require('./logger');
const fs = require('fs');
let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
@@ -5,11 +7,7 @@ const debugMode = process.env.YTDL_MODE === 'debug';
let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
var logger = null;
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_logger) {
setLogger(input_logger);
function initialize() {
ensureConfigFileExists();
ensureConfigItemsExist();
}
@@ -97,13 +95,13 @@ function getConfigItem(key) {
}
let path = CONFIG_ITEMS[key]['path'];
const val = Object.byString(config_json, path);
if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
return Object.byString(DEFAULT_CONFIG, path);
}
return Object.byString(config_json, path);
};
}
function setConfigItem(key, value) {
let success = false;
@@ -175,7 +173,7 @@ module.exports = {
globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
}
DEFAULT_CONFIG = {
const DEFAULT_CONFIG = {
"YoutubeDLMaterial": {
"Host": {
"url": "http://example.com",
@@ -184,25 +182,34 @@ DEFAULT_CONFIG = {
"Downloader": {
"path-audio": "audio/",
"path-video": "video/",
"default_file_output": "",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"enable_downloads_manager": true
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": ""
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",
@@ -211,7 +218,8 @@ DEFAULT_CONFIG = {
"Subscriptions": {
"allow_subscriptions": true,
"subscriptions_base_path": "subscriptions/",
"subscriptions_check_interval": "300"
"subscriptions_check_interval": "86400",
"redownload_fresh_uploads": false
},
"Users": {
"base_path": "users/",
@@ -225,7 +233,12 @@ DEFAULT_CONFIG = {
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": true,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,
"custom_downloading_agent": "",
"multi_user_mode": false,

75
backend/consts.js
View File

@@ -1,4 +1,4 @@
let CONFIG_ITEMS = {
exports.CONFIG_ITEMS = {
// Host
'ytdl_url': {
'key': 'ytdl_url',
@@ -18,6 +18,10 @@ let CONFIG_ITEMS = {
'key': 'ytdl_video_folder_path',
'path': 'YoutubeDLMaterial.Downloader.path-video'
},
'ytdl_default_file_output': {
'key': 'ytdl_default_file_output',
'path': 'YoutubeDLMaterial.Downloader.default_file_output'
},
'ytdl_use_youtubedl_archive': {
'key': 'ytdl_use_youtubedl_archive',
'path': 'YoutubeDLMaterial.Downloader.use_youtubedl_archive'
@@ -38,6 +42,14 @@ let CONFIG_ITEMS = {
'key': 'ytdl_include_metadata',
'path': 'YoutubeDLMaterial.Downloader.include_metadata'
},
'ytdl_max_concurrent_downloads': {
'key': 'ytdl_max_concurrent_downloads',
'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
},
'ytdl_download_rate_limit': {
'key': 'ytdl_download_rate_limit',
'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
},
// Extra
'ytdl_title_top': {
@@ -56,14 +68,18 @@ let CONFIG_ITEMS = {
'key': 'ytdl_download_only_mode',
'path': 'YoutubeDLMaterial.Extra.download_only_mode'
},
'ytdl_allow_multi_download_mode': {
'key': 'ytdl_allow_multi_download_mode',
'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
'ytdl_allow_autoplay': {
'key': 'ytdl_allow_autoplay',
'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
},
'ytdl_enable_downloads_manager': {
'key': 'ytdl_enable_downloads_manager',
'path': 'YoutubeDLMaterial.Extra.enable_downloads_manager'
},
'ytdl_allow_playlist_categorization': {
'key': 'ytdl_allow_playlist_categorization',
'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
},
// API
'ytdl_use_api_key': {
@@ -82,6 +98,27 @@ let CONFIG_ITEMS = {
'key': 'ytdl_youtube_api_key',
'path': 'YoutubeDLMaterial.API.youtube_API_key'
},
'ytdl_use_twitch_api': {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_api_key': {
'key': 'ytdl_twitch_api_key',
'path': 'YoutubeDLMaterial.API.twitch_API_key'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
},
'ytdl_use_sponsorblock_api': {
'key': 'ytdl_use_sponsorblock_api',
'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
},
'ytdl_generate_nfo_files': {
'key': 'ytdl_generate_nfo_files',
'path': 'YoutubeDLMaterial.API.generate_NFO_files'
},
// Themes
'ytdl_default_theme': {
@@ -106,9 +143,9 @@ let CONFIG_ITEMS = {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_check_interval': {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
'ytdl_subscriptions_redownload_fresh_uploads': {
'key': 'ytdl_subscriptions_redownload_fresh_uploads',
'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
},
// Users
@@ -129,7 +166,21 @@ let CONFIG_ITEMS = {
'path': 'YoutubeDLMaterial.Users.ldap_config'
},
// Database
'ytdl_use_local_db': {
'key': 'ytdl_use_local_db',
'path': 'YoutubeDLMaterial.Database.use_local_db'
},
'ytdl_mongodb_connection_string': {
'key': 'ytdl_mongodb_connection_string',
'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
},
// Advanced
'ytdl_default_downloader': {
'key': 'ytdl_default_downloader',
'path': 'YoutubeDLMaterial.Advanced.default_downloader'
},
'ytdl_use_default_downloading_agent': {
'key': 'ytdl_use_default_downloading_agent',
'path': 'YoutubeDLMaterial.Advanced.use_default_downloading_agent'
@@ -160,7 +211,7 @@ let CONFIG_ITEMS = {
}
};
AVAILABLE_PERMISSIONS = [
exports.AVAILABLE_PERMISSIONS = [
'filemanager',
'settings',
'subscriptions',
@@ -169,8 +220,6 @@ AVAILABLE_PERMISSIONS = [
'downloads_manager'
];
module.exports = {
CONFIG_ITEMS: CONFIG_ITEMS,
AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS,
CURRENT_VERSION: 'v4.1'
}
exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
exports.CURRENT_VERSION = 'v4.2';

966
backend/db.js
View File

File diff suppressed because it is too large Load Diff

632
backend/downloader.js Normal file
View File

@@ -0,0 +1,632 @@
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const glob = require('glob')
const Mutex = require('async-mutex').Mutex;
const youtubedl = require('youtube-dl');
const logger = require('./logger');
const config_api = require('./config');
const twitch_api = require('./twitch');
const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
let db_api = null;
const mutex = new Mutex();
let should_check_downloads = true;
const archivePath = path.join(__dirname, 'appdata', 'archives');
function setDB(input_db_api) { db_api = input_db_api }
exports.initialize = (input_db_api) => {
setDB(input_db_api);
categories_api.initialize(db_api);
if (db_api.database_initialized) {
setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
});
}
}
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
return await mutex.runExclusive(async () => {
const download = {
url: url,
type: type,
title: '',
user_uid: user_uid,
sub_id: sub_id,
sub_name: sub_name,
options: options,
uid: uuid(),
step_index: 0,
paused: false,
running: false,
finished_step: true,
error: null,
percent_complete: null,
finished: false,
timestamp_start: Date.now()
};
await db_api.insertRecordIntoTable('download_queue', download);
should_check_downloads = true;
return download;
});
}
exports.pauseDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
logger.warn(`Download ${download_uid} is already paused!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be paused before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
}
exports.resumeDownload = async (download_uid) => {
return await mutex.runExclusive(async () => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (!download['paused']) {
logger.warn(`Download ${download_uid} is not paused!`);
return false;
}
const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
should_check_downloads = true;
return success;
})
}
exports.restartDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await exports.clearDownload(download_uid);
const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
should_check_downloads = true;
return success;
}
exports.cancelDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['cancelled']) {
logger.warn(`Download ${download_uid} is already cancelled!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be cancelled before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
}
exports.clearDownload = async (download_uid) => {
return await db_api.removeRecord('download_queue', {uid: download_uid});
}
async function handleDownloadError(download_uid, error_message) {
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
}
async function setupDownloads() {
await fixDownloadState();
setInterval(checkDownloads, 1000);
}
async function fixDownloadState() {
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
for (let i = 0; i < running_downloads.length; i++) {
const running_download = running_downloads[i];
const update_obj = {finished_step: true, paused: true, running: false};
if (running_download['step_index'] > 0) {
update_obj['step_index'] = running_download['step_index'] - 1;
}
await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
}
}
async function checkDownloads() {
if (!should_check_downloads) return;
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
await mutex.runExclusive(async () => {
// avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
if (running_downloads.length === 0) {
should_check_downloads = false;
logger.verbose('Disabling checking downloads as none are available.');
}
return;
});
let running_downloads_count = downloads.filter(download => download['running']).length;
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
for (let i = 0; i < waiting_downloads.length; i++) {
const waiting_download = waiting_downloads[i];
const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
if (waiting_download['finished_step'] && !waiting_download['finished']) {
// move to next step
running_downloads_count++;
if (waiting_download['step_index'] === 0) {
collectInfo(waiting_download['uid']);
} else if (waiting_download['step_index'] === 1) {
downloadQueuedFile(waiting_download['uid']);
}
}
}
}
async function collectInfo(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Collecting info for download ${download_uid}`);
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
if (download['user_uid'] && !options.customFileFolderPath) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, download['user_uid'], type);
options.customFileFolderPath = user_path + path.sep;
}
let args = await exports.generateArgs(url, type, options, download['user_uid']);
// get video info prior to download
let info = await getVideoInfoByURL(url, args, download_uid);
if (!info) {
// info failed, error presumably already recorded
return;
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
info = await getVideoInfoByURL(url, args, download_uid);
}
// setup info required to calculate download progress
const expected_file_size = utils.getExpectedFileSize(info);
const files_to_check_for_progress = [];
// store info in download for future use
if (Array.isArray(info)) {
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
} else {
files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
}
const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
finished_step: true,
running: false,
options: options,
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title']
});
}
async function downloadQueuedFile(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Downloading ${download_uid}`);
return new Promise(async resolve => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
const args = download['args'];
const category = download['category'];
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
if (options.customFileFolderPath) {
fileFolderPath = options.customFileFolderPath;
}
fs.ensureDirSync(fileFolderPath);
const start_time = Date.now();
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
// download file
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
const file_objs = [];
let end_time = Date.now();
let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download_uid, err.stderr);
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download_uid, error_message);
logger.warn(error_message);
resolve(false);
return;
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
const ext = type === 'audio' ? '.mp3' : '.mp4';
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj);
}
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download_uid, error_message);
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
resolve();
}
});
});
}
// helper functions
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const globalArgs = config_api.getConfigItem('ytdl_custom_args');
const useCookies = config_api.getConfigItem('ytdl_use_cookies');
const is_audio = type === 'audio';
let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
const customArgs = options.customArgs;
let customOutput = options.customOutput;
const customQualityConfiguration = options.customQualityConfiguration;
// video-specific args
const selectedHeight = options.selectedHeight;
// audio-specific args
const maxBitrate = options.maxBitrate;
const youtubeUsername = options.youtubeUsername;
const youtubePassword = options.youtubePassword;
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
qualityPath = null;
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
qualityPath = ['-f', 'bestvideo+bestaudio']
}
if (customArgs) {
downloadConfig = customArgs.split(',,');
} else {
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
if (customOutput) {
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
if (qualityPath) downloadConfig.push(...qualityPath);
if (is_audio && !options.skip_audio_args) {
downloadConfig.push('-x');
downloadConfig.push('--audio-format', 'mp3');
}
if (youtubeUsername && youtubePassword) {
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
}
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
// if global args has an output, replce the original output with that of global args
const original_output_index = downloadConfig.indexOf('-o');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
}
if (options.additionalArgs && options.additionalArgs !== '') {
downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
}
// filter out incompatible args
downloadConfig = filterArgs(downloadConfig, is_audio);
if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
return downloadConfig;
}
async function getVideoInfoByURL(url, args = [], download_uid = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
new_args.push('--dump-json');
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
if (output) {
let outputs = [];
try {
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
outputs.push(output_json);
}
resolve(outputs.length === 1 ? outputs[0] : outputs);
} catch(e) {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
await handleDownloadError(download_uid, error);
}
resolve(null);
}
} else {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
await handleDownloadError(download_uid, error_message);
}
resolve(null);
}
});
});
}
function filterArgs(args, isAudio) {
const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
const args_to_remove = isAudio ? video_only_args : audio_only_args;
return args.filter(x => !args_to_remove.includes(x));
}
async function checkDownloadPercent(download_uid) {
/*
This is more of an art than a science, we're just selecting files that start with the file name,
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
be divided by the "total expected bytes."
*/
const download = await db_api.getRecord('download_queue', {uid: download_uid});
const files_to_check_for_progress = download['files_to_check_for_progress'];
const resulting_file_size = download['expected_file_size'];
if (!resulting_file_size) return;
let sum_size = 0;
glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
files.forEach(async file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
});
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
});
}
exports.generateNFOFile = (info, output_path) => {
const nfo_obj = {
episodedetails: {
title: info['fulltitle'],
episode: info['playlist_index'] ? info['playlist_index'] : undefined,
premiered: utils.formatDateString(info['upload_date']),
plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
director: info['artist'] ? info['artist'] : info['uploader']
}
};
const doc = create(nfo_obj);
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join(archivePath);
}
}

8
backend/ecosystem.config.js Normal file
View File

@@ -0,0 +1,8 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder",
watch_delay: 5000
}]
}

2
backend/entrypoint.sh
View File

@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
CMD="node app.js"
CMD="pm2-runtime pm2.config.js"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then

23
backend/logger.js Normal file
View File

@@ -0,0 +1,23 @@
const winston = require('winston');
let debugMode = process.env.YTDL_MODE === 'debug';
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
]
});
module.exports = logger;

1522
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

17
backend/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon -q app.js"
"start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
},
"nodemonConfig": {
"ignore": [
@@ -14,7 +15,8 @@
"public/*"
],
"watch": [
"restart.json"
"restart_update.json",
"restart_general.json"
]
},
"repository": {
@@ -30,6 +32,8 @@
"dependencies": {
"archiver": "^3.1.1",
"async": "^3.1.0",
"async-mutex": "^0.3.1",
"axios": "^0.21.2",
"bcryptjs": "^2.4.0",
"compression": "^1.7.4",
"config": "^3.2.3",
@@ -42,10 +46,13 @@
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^8.4.0",
"moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"node-id3": "^0.1.14",
"nodemon": "^2.0.2",
"nodemon": "^2.0.7",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",
@@ -54,10 +61,12 @@
"progress": "^2.0.3",
"ps-node": "^0.1.6",
"read-last-lines": "^1.7.2",
"rxjs": "^7.3.0",
"shortid": "^2.2.15",
"unzipper": "^0.10.10",
"uuidv4": "^6.0.6",
"winston": "^3.2.1",
"winston": "^3.3.3",
"xmlbuilder2": "^3.0.2",
"youtube-dl": "^3.0.2"
}
}

7
backend/pm2.config.js Normal file
View File

@@ -0,0 +1,7 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder"
}]
}

500
backend/subscriptions.js
View File

@@ -1,26 +1,21 @@
const FileSync = require('lowdb/adapters/FileSync')
const fs = require('fs-extra');
const path = require('path');
const youtubedl = require('youtube-dl');
var fs = require('fs-extra');
const { uuid } = require('uuidv4');
var path = require('path');
var youtubedl = require('youtube-dl');
const config_api = require('./config');
var utils = require('./utils')
const utils = require('./utils');
const logger = require('./logger');
const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
let db_api = null;
let downloader_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
function initialize(input_db_api, input_downloader_api) {
setDB(input_db_api);
downloader_api = input_downloader_api;
}
async function subscribe(sub, user_uid = null) {
@@ -33,12 +28,7 @@ async function subscribe(sub, user_uid = null) {
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = false;
if (user_uid)
url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
else
url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
if (!sub.name && url_exists) {
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -47,23 +37,16 @@ async function subscribe(sub, user_uid = null) {
return;
}
// add sub to db
let sub_db = null;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
let success = await getSubscriptionInfo(sub, user_uid);
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub);
if (success) {
sub = sub_db.value();
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
};
}
result_obj.success = success;
result_obj.sub = sub;
@@ -72,13 +55,7 @@ async function subscribe(sub, user_uid = null) {
}
async function getSubscriptionInfo(sub, user_uid = null) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
async function getSubscriptionInfo(sub) {
// get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
@@ -90,8 +67,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id);
}
@@ -114,34 +91,17 @@ async function getSubscriptionInfo(sub, user_uid = null) {
continue;
}
if (!sub.name) {
sub.name = sub.isPlaylist ? output_json.playlist_title : output_json.uploader;
if (sub.isPlaylist) {
sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
} else {
sub.name = output_json.uploader;
}
// if it's now valid, update
if (sub.name) {
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
}
}
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !sub.archive) {
// must create the archive
const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
const archive_path = path.join(archive_dir, 'archive.txt');
// creates archive directory and text file if it doesn't exist
fs.ensureDirSync(archive_dir);
fs.ensureFileSync(archive_path);
// updates subscription
sub.archive = archive_dir;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
}
// TODO: get even more info
resolve(true);
@@ -158,13 +118,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let result_obj = { success: false, error: '' };
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
const sub_files = await db_api.getRecords('files', {sub_id: id});
for (let i = 0; i < sub_files.length; i++) {
const sub_file = sub_files[i];
if (config_api.descriptors[sub_file['uid']]) {
try {
for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
config_api.descriptors[sub_file['uid']][i].destroy();
}
} catch(e) {
continue;
}
}
}
await db_api.removeRecord('subscriptions', {id: id});
await db_api.removeAllRecords('files', {sub_id: id});
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
@@ -186,27 +158,23 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
// TODO: combine this with deletefile
let basePath = null;
let sub_db = null;
if (user_uid) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
sub_db = db.get('subscriptions').find({id: sub.id});
}
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
await db_api.removeRecord('files', {uid: file_uid});
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.webp');
const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
fs.pathExists(jsonPath),
@@ -238,7 +206,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
utils.removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
@@ -250,17 +218,125 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
}
async function getVideosForSub(sub, user_uid = null) {
if (!subExists(sub.id, user_uid)) {
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
updateSubscriptionProperty(sub, {downloading: true}, user_uid);
// get basePath
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appendedBasePath = getAppendedBasePath(sub, basePath);
fs.ensureDirSync(appendedBasePath);
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
// TODO: reimplement
// const outputs = err.stdout.split(/\r\n|\r|\n/);
// for (let i = 0; i < outputs.length; i++) {
// const output = JSON.parse(outputs[i]);
// await handleOutputJSON(sub, output, i === 0, multiUserMode)
// if (err.stderr.includes(output['id']) && archive_path) {
// // we found a video that errored! add it to the archive to prevent future errors
// if (sub.archive) {
// archive_dir = sub.archive;
// archive_path = path.join(archive_dir, 'archive.txt')
// fs.appendFileSync(archive_path, output['id']);
// }
// }
// }
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name);
resolve(true);
return;
}
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
output_jsons.push(output_json);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
}
resolve(files_to_download);
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
}
resolve(true);
}
});
}, err => {
logger.error(err);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
});
}
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const base_download_options = {
selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
customFileFolderPath: getAppendedBasePath(sub, basePath),
customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
additionalArgs: sub.custom_args
}
return base_download_options;
}
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
// get basePath
let basePath = null;
if (user_uid)
@@ -270,25 +346,18 @@ async function getVideosForSub(sub, user_uid = null) {
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = null
appendedBasePath = getAppendedBasePath(sub, basePath);
let appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
if (sub.custom_output) {
let fullOutput = `${appendedBasePath}/${file_output}.%(ext)s`;
if (desired_path) {
fullOutput = `${desired_path}.%(ext)s`;
} else if (sub.custom_output) {
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
@@ -296,13 +365,14 @@ async function getVideosForSub(sub, user_uid = null) {
qualityPath.push('-x');
qualityPath.push('--audio-format', 'mp3');
} else {
qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4']
if (!sub.maxQuality || sub.maxQuality === 'best') qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'];
else qualityPath = ['-f', `bestvideo[height<=${sub.maxQuality}]+bestaudio/best[height<=${sub.maxQuality}]`, '--merge-output-format', 'mp4'];
}
downloadConfig.push(...qualityPath)
if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,');
const customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
@@ -314,7 +384,7 @@ async function getVideosForSub(sub, user_uid = null) {
let archive_dir = null;
let archive_path = null;
if (useArchive) {
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
@@ -327,7 +397,7 @@ async function getVideosForSub(sub, user_uid = null) {
downloadConfig = ['-f', 'best', '--dump-json'];
}
if (sub.timerange) {
if (sub.timerange && !redownload) {
downloadConfig.push('--dateafter', sub.timerange);
}
@@ -344,151 +414,135 @@ async function getVideosForSub(sub, user_uid = null) {
downloadConfig.push('--write-thumbnail');
}
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
fs.appendFileSync(archive_path, output['id']);
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name);
resolve(true);
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
return downloadConfig;
}
// TODO: Potentially store downloaded files in db?
}
resolve(true);
async function getFilesToDownload(sub, output_jsons) {
const files_to_download = [];
for (let i = 0; i < output_jsons.length; i++) {
const output_json = output_jsons[i];
const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
if (file_missing) {
const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
}
});
}, err => {
logger.error(err);
});
}
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
files_to_download.push(output_json);
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
} else {
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
}
return files_to_download;
}
function getAllSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
else
return db.get('subscriptions').value();
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
function getSubscription(subID, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return db.get('subscriptions').find({id: subID}).value();
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}
function updateSubscription(sub, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(sub).write();
}
async function getSubscription(subID) {
return await db_api.getRecord('subscriptions', {id: subID});
}
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
async function updateSubscription(sub) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
return true;
}
function subExists(subID, user_uid = null) {
if (user_uid)
return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return !!db.get('subscriptions').find({id: subID}).value();
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
});
}
async function updateSubscriptionProperty(sub, assignment_obj) {
// TODO: combine with updateSubscription
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
return true;
}
async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (current_date === video['upload_date'].replace(/-/g, '')) {
// set upload as fresh
const video_uid = video['uid'];
await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
}
});
}
async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
await checkVideoIfBetterExists(video, sub, user_uid)
}
});
}
async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
if (err) {
// video is not available anymore for whatever reason
} else if (output) {
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
if (output[metric_to_compare] > file_obj[metric_to_compare]) {
// download new video as the simulated one is better
youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (err) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (output) {
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
}
});
}
}
});
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
}
// helper functions
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
getSubscriptions : getSubscriptions,
getAllSubscriptions : getAllSubscriptions,
updateSubscription : updateSubscription,
subscribe : subscribe,
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
removeIDFromArchive : removeIDFromArchive,
setLogger : setLogger,
initialize : initialize
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
}

1
backend/test/sample.info.json Normal file
View File

File diff suppressed because one or more lines are too long

352
backend/test/tests.js Normal file
View File

@@ -0,0 +1,352 @@
var assert = require('assert');
const low = require('lowdb')
var winston = require('winston');
process.chdir('./backend')
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync('./appdata/db.json');
const db = low(adapter)
const users_adapter = new FileSync('./appdata/users.json');
const users_db = low(users_adapter);
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
let debugMode = process.env.YTDL_MODE === 'debug';
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: 'debug', name: 'console'})
]
});
var auth_api = require('../authentication/auth');
var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db);
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(true);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Transfer to local', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(false);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
});
describe('Export', function() {
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon: http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
});
describe('Multi User', async function() {
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
});
describe('Authentication', function() {
it('login', async function() {
assert(user);
});
});
describe('Video player - normal', function() {
const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
it('Get video', async function() {
const video_obj = db_api.getVideo(video_to_test, 'admin');
assert(video_obj);
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
const playlist_files_to_download = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const playlist_file = await db_api.getVideo(uid, user_to_test);
playlist_files_to_download.push(playlist_file);
}
const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
it('Subscription zip generator', async function() {
const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
assert(sub);
const sub_files_to_download = [];
for (let i = 0; i < sub_videos.length; i++) {
const sub_file = sub_videos[i];
sub_files_to_download.push(sub_file);
}
const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
});
// describe('Video player - subscription', function() {
// const sub_to_test = '';
// const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
// it('Get video', async function() {
// const video_obj = db_api.getVideo(video_to_test, 'admin', );
// assert(video_obj);
// });
// it('Video access - disallowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(!video_obj);
// });
// it('Video access - allowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(video_obj);
// });
// });
});
describe('Downloader', function() {
const downloader_api = require('../downloader');
downloader_api.initialize(db_api);
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const options = {
ui_uid: uuid(),
user: 'admin'
}
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('download_queue');
});
it('Get file info', async function() {
});
it('Download file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Queue file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Pause file', async function() {
});
it('Generate args', async function() {
const args = await downloader_api.generateArgs(url, 'video', options);
console.log(args);
});
it('Generate args - subscription', async function() {
subscriptions_api.initialize(db_api, logger);
const sub = await subscriptions_api.getSubscription(sub_id);
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
console.log(args);
});
it('Generate kodi NFO file', async function() {
const nfo_file_path = './test/sample.nfo';
if (fs.existsSync(nfo_file_path)) {
fs.unlinkSync(nfo_file_path);
}
const sample_json = fs.readJSONSync('./test/sample.info.json');
downloader_api.generateNFOFile(sample_json, nfo_file_path);
assert(fs.existsSync(nfo_file_path), true);
});
});

128
backend/twitch.js Normal file
View File

@@ -0,0 +1,128 @@
var moment = require('moment');
var Axios = require('axios');
var fs = require('fs-extra')
var path = require('path');
const config_api = require('./config');
async function getCommentsForVOD(clientID, vodId) {
let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
batch,
cursor;
let comments = null;
try {
do {
batch = (await Axios.get(url, {
headers: {
'Client-ID': clientID,
Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
'Content-Type': 'application/json; charset=UTF-8',
}
})).data;
const str = batch.comments.map(c => {
let {
created_at: msgCreated,
content_offset_seconds: timestamp,
commenter: {
name,
_id,
created_at: acctCreated
},
message: {
body: msg,
user_color: user_color
}
} = c;
const timestamp_str = moment.duration(timestamp, 'seconds')
.toISOString()
.replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
(_, ...ms) => {
const seg = v => v ? v.padStart(2, '0') : '00';
return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
});
acctCreated = moment(acctCreated).utc();
msgCreated = moment(msgCreated).utc();
if (!comments) comments = [];
comments.push({
timestamp: timestamp,
timestamp_str: timestamp_str,
name: name,
message: msg,
user_color: user_color
});
// let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
// return line;
}).join('\n');
cursor = batch._next;
url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
await new Promise(res => setTimeout(res, 300));
} while (cursor);
} catch (err) {
console.error(err);
}
return comments;
}
async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
let file_path = null;
if (user_uid) {
if (sub) {
file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
} else {
file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
}
} else {
if (sub) {
file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
} else {
file_path = path.join(type, id + '.twitch_chat.json');
}
}
var chat_file = null;
if (fs.existsSync(file_path)) {
chat_file = fs.readJSONSync(file_path);
}
return chat_file;
}
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
const chat = await getCommentsForVOD(twitch_api_key, vodId);
// save file if needed params are included
let file_path = null;
if (user_uid) {
if (sub) {
file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
} else {
file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
}
} else {
if (sub) {
file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
} else {
file_path = path.join(type, id + '.twitch_chat.json');
}
}
if (chat) fs.writeJSONSync(file_path, chat);
return chat;
}
module.exports = {
getCommentsForVOD: getCommentsForVOD,
getTwitchChatByFileID: getTwitchChatByFileID,
downloadTwitchChatByVODID: downloadTwitchChatByVODID
}

274
backend/utils.js
View File

@@ -1,6 +1,10 @@
var fs = require('fs-extra')
var path = require('path')
const fs = require('fs-extra')
const path = require('path')
const ffmpeg = require('fluent-ffmpeg');
const config_api = require('./config');
const logger = require('./logger');
const CONSTS = require('./consts')
const archiver = require('archiver');
const is_windows = process.platform === 'win32';
@@ -20,7 +24,7 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path;
}
async function getDownloadedFilesByType(basePath, type) {
async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
// return empty array if the path doesn't exist
if (!(await fs.pathExists(basePath))) return [];
@@ -36,23 +40,58 @@ async function getDownloadedFilesByType(basePath, type) {
var id = file_path.substring(0, file_path.length-4);
var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var size = stats.size;
if (full_metadata) {
jsonobj['id'] = id;
files.push(jsonobj);
continue;
}
var upload_date = formatDateString(jsonobj.upload_date);
var isaudio = type === 'audio';
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
files.push(file_obj);
}
return files;
}
async function createContainerZipFile(container_obj, container_file_objs) {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
}
async function createZipFile(zip_file_path, file_paths) {
let output = fs.createWriteStream(zip_file_path);
var archive = archiver('zip', {
gzip: true,
zlib: { level: 9 } // Sets the compression level.
});
archive.on('error', function(err) {
logger.error(err);
throw err;
});
// pipe archive data to the output file
archive.pipe(output);
for (let file_path of file_paths) {
const file_name = path.parse(file_path).base;
archive.file(file_path, {name: file_name})
}
await archive.finalize();
// wait a tiny bit for the zip to reload in fs
await wait(100);
return zip_file_path;
}
function getJSONMp4(name, customPath, openReadPerms = false) {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -85,16 +124,31 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj;
}
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
}
else obj = 0;
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
function getDownloadedThumbnail(file_path) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
let pngPath = file_path_no_extension + '.png';
if (fs.existsSync(jpgPath))
return jpgPath;
@@ -106,38 +160,41 @@ function getDownloadedThumbnail(name, type, customPath = null) {
return null;
}
function getExpectedFileSize(info_json) {
if (info_json['filesize']) {
return info_json['filesize'];
}
function getExpectedFileSize(input_info_jsons) {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
const formats = info_json['format_id'].split('+');
let expected_filesize = 0;
formats.forEach(format_id => {
info_json.formats.forEach(available_format => {
if (available_format.format_id === format_id && available_format.filesize) {
expected_filesize += available_format.filesize;
}
info_jsons.forEach(info_json => {
const formats = info_json['format_id'].split('+');
let individual_expected_filesize = 0;
formats.forEach(format_id => {
info_json.formats.forEach(available_format => {
if (available_format.format_id === format_id && available_format.filesize) {
individual_expected_filesize += available_format.filesize;
}
});
});
expected_filesize += individual_expected_filesize;
});
return expected_filesize;
}
function fixVideoMetadataPerms(name, type, customPath = null) {
function fixVideoMetadataPerms(file_path, type) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
path.join(customPath, name + ext + '.info.json'),
file_path_no_extension + '.info.json',
file_path_no_extension + ext + '.info.json',
// Thumbnails
path.join(customPath, name + '.webp'),
path.join(customPath, name + '.jpg')
file_path_no_extension + '.webp',
file_path_no_extension + '.jpg'
];
for (const file of files_to_fix) {
@@ -146,18 +203,68 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
}
}
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
function deleteJSONFile(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let json_path = path.join(customPath, name + '.info.json');
let alternate_json_path = path.join(customPath, name + ext + '.info.json');
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
}
function durationStringToNumber(dur_str) {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
for (let i = dur_str_parts.length-1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
}
return num_sum;
}
function getMatchingCategoryFiles(category, files) {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
function getCurrentDownloader() {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
return details_json['downloader'];
}
async function recFindByExt(base,ext,files,result)
{
@@ -181,9 +288,76 @@ async function recFindByExt(base,ext,files,result)
return result
}
function removeFileExtension(filename) {
const filename_parts = filename.split('.');
filename_parts.splice(filename_parts.length - 1);
return filename_parts.join('.');
}
function formatDateString(date_string) {
return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
}
function createEdgeNGrams(str) {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
return str.split(" ").reduce((ngrams, token) => {
if (token.length > minGram) {
for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
ngrams = [...ngrams, token.substr(0, i)]
}
} else {
ngrams = [...ngrams, token]
}
return ngrams
}, []).join(" ")
}
return str
}
// ffmpeg helper functions
async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
let base_ffmpeg_call = ffmpeg(file_path);
if (start) {
base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
}
if (end) {
base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
}
base_ffmpeg_call
.on('end', () => {
logger.verbose(`Cropping for '${file_path}' complete.`);
fs.unlinkSync(file_path);
fs.moveSync(temp_file_path, file_path);
resolve(true);
})
.on('error', (err) => {
logger.error(`Failed to crop ${file_path}.`);
logger.error(err);
resolve(false);
}).save(temp_file_path);
});
}
/**
* setTimeout, but its a promise.
* @param {number} ms
*/
async function wait(ms) {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) {
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
this.id = id;
this.title = title;
this.thumbnailURL = thumbnailURL;
@@ -194,17 +368,33 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
this.size = size;
this.path = path;
this.upload_date = upload_date;
this.description = description;
this.view_count = view_count;
this.height = height;
this.abr = abr;
}
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive: removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
File: File
}

23
chart/.helmignore Normal file
View File

@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
chart/Chart.yaml Normal file
View File

@@ -0,0 +1,24 @@
apiVersion: v2
name: youtubedl-material
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "4.2"

22
chart/templates/NOTES.txt Normal file
View File

@@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "youtubedl-material.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "youtubedl-material.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "youtubedl-material.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "youtubedl-material.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}

62
chart/templates/_helpers.tpl Normal file
View File

@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "youtubedl-material.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "youtubedl-material.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "youtubedl-material.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "youtubedl-material.labels" -}}
helm.sh/chart: {{ include "youtubedl-material.chart" . }}
{{ include "youtubedl-material.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "youtubedl-material.selectorLabels" -}}
app.kubernetes.io/name: {{ include "youtubedl-material.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "youtubedl-material.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "youtubedl-material.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

21
chart/templates/appdata-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.appdata.enabled (not .Values.persistence.appdata.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-appdata
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.appdata.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.appdata.size | quote }}
{{- if .Values.persistence.appdata.storageClass }}
{{- if (eq "-" .Values.persistence.appdata.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.appdata.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

21
chart/templates/audio-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.audio.enabled (not .Values.persistence.audio.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-audio
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.audio.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.audio.size | quote }}
{{- if .Values.persistence.audio.storageClass }}
{{- if (eq "-" .Values.persistence.audio.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.audio.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

121
chart/templates/deployment.yaml Normal file
View File

@@ -0,0 +1,121 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "youtubedl-material.fullname" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "youtubedl-material.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "youtubedl-material.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "youtubedl-material.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: 17442
protocol: TCP
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- mountPath: /app/appdata
name: appdata
{{- if .Values.persistence.appdata.subPath }}
subPath: {{ .Values.persistence.appdata.subPath }}
{{- end }}
- mountPath: /app/audio
name: audio
{{- if .Values.persistence.audio.subPath }}
subPath: {{ .Values.persistence.audio.subPath }}
{{- end }}
- mountPath: /app/video
name: video
{{- if .Values.persistence.video.subPath }}
subPath: {{ .Values.persistence.video.subPath }}
{{- end }}
- mountPath: /app/subscriptions
name: subscriptions
{{- if .Values.persistence.subscriptions.subPath }}
subPath: {{ .Values.persistence.subscriptions.subPath }}
{{- end }}
- mountPath: /app/users
name: users
{{- if .Values.persistence.users.subPath }}
subPath: {{ .Values.persistence.users.subPath }}
{{- end }}
volumes:
- name: appdata
{{- if .Values.persistence.appdata.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.appdata.existingClaim }}{{ .Values.persistence.appdata.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-appdata{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: audio
{{- if .Values.persistence.audio.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.audio.existingClaim }}{{ .Values.persistence.audio.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-audio{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: subscriptions
{{- if .Values.persistence.subscriptions.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.subscriptions.existingClaim }}{{ .Values.persistence.subscriptions.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-subscriptions{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: users
{{- if .Values.persistence.users.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.users.existingClaim }}{{ .Values.persistence.users.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-users{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: video
{{- if .Values.persistence.video.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.video.existingClaim }}{{ .Values.persistence.video.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-video{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

41
chart/templates/ingress.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "youtubedl-material.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
backend:
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}

15
chart/templates/service.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "youtubedl-material.fullname" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "youtubedl-material.selectorLabels" . | nindent 4 }}

12
chart/templates/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "youtubedl-material.serviceAccountName" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

21
chart/templates/subscriptions-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.subscriptions.enabled (not .Values.persistence.subscriptions.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-subscriptions
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.subscriptions.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.subscriptions.size | quote }}
{{- if .Values.persistence.subscriptions.storageClass }}
{{- if (eq "-" .Values.persistence.subscriptions.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.subscriptions.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

15
chart/templates/tests/test-connection.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "youtubedl-material.fullname" . }}-test-connection"
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "youtubedl-material.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never

21
chart/templates/users-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.users.enabled (not .Values.persistence.users.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-users
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.users.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.users.size | quote }}
{{- if .Values.persistence.users.storageClass }}
{{- if (eq "-" .Values.persistence.users.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.users.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

21
chart/templates/video-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.video.enabled (not .Values.persistence.video.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-video
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.video.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.video.size | quote }}
{{- if .Values.persistence.video.storageClass }}
{{- if (eq "-" .Values.persistence.video.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.video.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

153
chart/values.yaml Normal file
View File

@@ -0,0 +1,153 @@
# Default values for youtubedl-material.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: tzahi12345/youtubedl-material
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 17442
ingress:
enabled: false
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: chart-example.local
paths: []
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
persistence:
appdata:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 1Gi
audio:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
video:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
subscriptions:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
users:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
nodeSelector: {}
tolerations: []
affinity: {}

20
chrome-extension/background.js
View File

@@ -1,20 +0,0 @@
// background.js
// Called when the user clicks on the browser action.
chrome.browserAction.onClicked.addListener(function(tab) {
// get the frontend_url
chrome.storage.sync.get({
frontend_url: 'http://localhost',
audio_only: false
}, function(items) {
chrome.tabs.query({active: true, currentWindow: true}, function(tabs) {
var activeTab = tabs[0];
var url = activeTab.url;
if (url.includes('youtube.com')) {
var new_url = items.frontend_url + '/#/home;url=' + encodeURIComponent(url) + ';audioOnly=' + items.audio_only;
chrome.tabs.create({ url: new_url });
}
});
});
});

12
chrome-extension/manifest.json
View File

@@ -1,17 +1,17 @@
{
"manifest_version": 2,
"name": "YoutubeDL-Material",
"version": "0.3",
"version": "0.4",
"description": "The Official Firefox & Chrome Extension of YoutubeDL-Material, an open-source and self-hosted YouTube downloader.",
"background": {
"scripts": ["background.js"]
},
"browser_action": {
"default_icon": "favicon.png"
"default_icon": "favicon.png",
"default_popup": "popup.html",
"default_title": "YoutubeDL-Material"
},
"permissions": [
"tabs",
"storage"
"storage",
"contextMenus"
],
"options_ui": {
"page": "options.html",

35
chrome-extension/popup.html Normal file
View File

@@ -0,0 +1,35 @@
<!DOCTYPE html>
<html>
<head>
<!-- Scripts -->
<script src="js/jquery-3.4.1.min.js"></script>
<script src="js/popper.min.js"></script>
<script src="js/bootstrap.min.js"></script>
<!-- Cascading Style Sheets -->
<link href="css/bootstrap.min.css" rel="stylesheet" media="screen">
</head>
<body>
<div style="width: 400px; margin: 0 auto;">
<div style="margin: 10px;">
<div class="checkbox">
<label>
<input type="checkbox" id="audio_only">
Audio only
</label>
</div>
<div class="input-group mb-3">
<input id="url_input" type="text" class="form-control" placeholder="URL" aria-label="URL" aria-describedby="basic-addon2">
<div class="input-group-append">
<button class="btn btn-outline-secondary" type="button" id="download">Download</button>
</div>
</div>
</div>
</div>
<script src="popup.js"></script>
</body>
</html>

50
chrome-extension/popup.js Normal file
View File

@@ -0,0 +1,50 @@
function audioOnlyClicked() {
console.log('audio only clicked');
var audio_only = document.getElementById("audio_only").checked;
// save state
chrome.storage.sync.set({
audio_only: audio_only
}, function() {});
}
function downloadVideo() {
var input_url = document.getElementById("url_input").value
// get the frontend_url
chrome.storage.sync.get({
frontend_url: 'http://localhost',
audio_only: false
}, function(items) {
var download_url = items.frontend_url + '/#/home;url=' + encodeURIComponent(input_url) + ';audioOnly=' + items.audio_only;
chrome.tabs.create({ url: download_url });
});
}
function loadInputs() {
// load audio-only input
chrome.storage.sync.get({
frontend_url: 'http://localhost',
audio_only: false
}, function(items) {
document.getElementById("audio_only").checked = items.audio_only;
});
// load url input
chrome.tabs.query({active: true, currentWindow: true}, function(tabs) {
var activeTab = tabs[0];
var current_url = activeTab.url;
console.log(current_url);
if (current_url && current_url.includes('youtube.com')) {
document.getElementById("url_input").value = current_url;
}
});
}
document.getElementById('download').addEventListener('click',
downloadVideo);
document.getElementById('audio_only').addEventListener('click',
audioOnlyClicked);
document.addEventListener('DOMContentLoaded', loadInputs);

BIN
chrome-extension/youtubedl-material-chrome-extension.zip
View File

Binary file not shown.

BIN
chrome-extension/youtubedl-material-firefox-extension.zip
View File

Binary file not shown.

14
docker-compose.yml
View File

@@ -3,6 +3,9 @@ services:
ytdl_material:
environment:
ALLOW_CONFIG_MUTATIONS: 'true'
ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
ytdl_use_local_db: 'false'
write_ytdl_config: 'true'
restart: always
volumes:
- ./appdata:/app/appdata
@@ -12,4 +15,13 @@ services:
- ./users:/app/users
ports:
- "8998:17442"
image: tzahi12345/youtubedl-material:latest
image: tzahi12345/youtubedl-material:nightly
ytdl-mongo-db:
image: mongo
ports:
- "27017:27017"
logging:
driver: "none"
container_name: mongo-db
volumes:
- ./db/:/data/db

14107
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

76
package.json
View File

@@ -1,11 +1,12 @@
{
"name": "youtube-dl-material",
"version": "4.1.0",
"version": "4.2.0",
"license": "MIT",
"scripts": {
"ng": "ng",
"start": "ng serve",
"build": "ng build",
"build": "ng build --prod",
"prebuild": "node src/postbuild.mjs",
"heroku-postbuild": "npm install --prefix backend",
"test": "ng test",
"lint": "ng lint",
@@ -20,57 +21,64 @@
},
"private": true,
"dependencies": {
"@angular-devkit/core": "^9.0.6",
"@angular/animations": "^9.1.0",
"@angular/cdk": "^9.2.0",
"@angular/common": "^9.1.0",
"@angular/compiler": "^9.1.0",
"@angular/core": "^9.0.7",
"@angular/forms": "^9.1.0",
"@angular/localize": "^9.1.0",
"@angular/material": "^9.2.0",
"@angular/platform-browser": "^9.1.0",
"@angular/platform-browser-dynamic": "^9.1.0",
"@angular/router": "^9.1.0",
"@angular-devkit/core": "^11.0.4",
"@angular/animations": "^11.0.4",
"@angular/cdk": "^11.0.2",
"@angular/common": "^11.0.4",
"@angular/compiler": "^11.0.4",
"@angular/core": "^11.0.4",
"@angular/forms": "^11.0.4",
"@angular/localize": "^11.0.4",
"@angular/material": "^11.0.2",
"@angular/platform-browser": "^11.0.4",
"@angular/platform-browser-dynamic": "^11.0.4",
"@angular/router": "^11.0.4",
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^2.1.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
"filesize": "^6.1.0",
"fingerprintjs2": "^2.1.0",
"fs-extra": "^10.0.0",
"material-icons": "^0.5.4",
"nan": "^2.14.1",
"ng-lazyload-image": "^7.0.1",
"ngx-avatar": "^4.0.0",
"ngx-file-drop": "^9.0.1",
"ngx-videogular": "^9.0.1",
"rxjs": "^6.5.3",
"rxjs": "^6.6.3",
"rxjs-compat": "^6.0.0-rc.0",
"tslib": "^1.10.0",
"typescript": "~3.7.5",
"tslib": "^2.0.0",
"typescript": "~4.0.5",
"web-animations-js": "^2.3.2",
"xliff-to-json": "^1.0.4",
"zone.js": "~0.10.2"
},
"devDependencies": {
"@angular-devkit/build-angular": "^0.901.0",
"@angular/cli": "^9.0.7",
"@angular/compiler-cli": "^9.0.7",
"@angular/language-service": "^9.0.7",
"@angular-devkit/build-angular": "^0.1100.4",
"@angular/cli": "^11.0.4",
"@angular/compiler-cli": "^11.0.4",
"@angular/language-service": "^11.0.4",
"@types/core-js": "^2.5.2",
"@types/file-saver": "^2.0.1",
"@types/jasmine": "2.5.45",
"@types/jasmine": "~3.6.0",
"@types/node": "^12.11.1",
"codelyzer": "^5.1.2",
"electron": "^8.0.1",
"jasmine-core": "~2.6.2",
"jasmine-spec-reporter": "~4.1.0",
"karma": "~1.7.0",
"karma-chrome-launcher": "~2.1.1",
"@typescript-eslint/eslint-plugin": "^4.29.0",
"@typescript-eslint/parser": "^4.29.0",
"codelyzer": "^6.0.0",
"electron": "^9.4.0",
"eslint": "^7.32.0",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
"karma": "~5.0.0",
"karma-chrome-launcher": "~3.1.0",
"karma-cli": "~1.0.1",
"karma-coverage-istanbul-reporter": "^1.2.1",
"karma-jasmine": "~1.1.0",
"karma-jasmine-html-reporter": "^0.2.2",
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"openapi-typescript-codegen": "^0.4.11",
"protractor": "~5.1.2",
"protractor": "~7.0.0",
"ts-node": "~3.0.4",
"tslint": "~5.3.2"
"tslint": "~6.1.0"
}
}

6
src/app/app-routing.module.ts
View File

@@ -7,19 +7,21 @@ import { SubscriptionComponent } from './subscription/subscription/subscription.
import { PostsService } from './posts.services';
import { LoginComponent } from './components/login/login.component';
import { DownloadsComponent } from './components/downloads/downloads.component';
import { SettingsComponent } from './settings/settings.component';
const routes: Routes = [
{ path: 'home', component: MainComponent, canActivate: [PostsService] },
{ path: 'player', component: PlayerComponent, canActivate: [PostsService]},
{ path: 'subscriptions', component: SubscriptionsComponent, canActivate: [PostsService] },
{ path: 'subscription', component: SubscriptionComponent, canActivate: [PostsService] },
{ path: 'settings', component: SettingsComponent, canActivate: [PostsService] },
{ path: 'login', component: LoginComponent },
{ path: 'downloads', component: DownloadsComponent },
{ path: 'downloads', component: DownloadsComponent, canActivate: [PostsService] },
{ path: '', redirectTo: '/home', pathMatch: 'full' }
];
@NgModule({
imports: [RouterModule.forRoot(routes, { useHash: true })],
imports: [RouterModule.forRoot(routes, { useHash: true, relativeLinkResolution: 'legacy' })],
exports: [RouterModule]
})
export class AppRoutingModule { }

14
src/app/app.component.html
View File

@@ -23,10 +23,10 @@
<span i18n="Dark mode toggle label">Dark</span>
<mat-slide-toggle class="theme-slide-toggle" [checked]="postsService.theme.key === 'dark'"></mat-slide-toggle>
</button>
<button *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('settings')))" (click)="openSettingsDialog()" mat-menu-item>
<!-- <button *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('settings')))" (click)="openSettingsDialog()" mat-menu-item>
<mat-icon>settings</mat-icon>
<span i18n="Settings menu label">Settings</span>
</button>
</button> -->
<button (click)="openAboutDialog()" mat-menu-item>
<mat-icon>info</mat-icon>
<span i18n="About menu label">About</span>
@@ -42,10 +42,14 @@
<mat-nav-list>
<a *ngIf="postsService.config && (!postsService.config.Advanced.multi_user_mode || postsService.isLoggedIn)" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/home'><ng-container i18n="Navigation menu Home Page title">Home</ng-container></a>
<a *ngIf="postsService.config && postsService.config.Advanced.multi_user_mode && !postsService.isLoggedIn" mat-list-item (click)="sidenav.close()" routerLink='/login'><ng-container i18n="Navigation menu Login Page title">Login</ng-container></a>
<a *ngIf="postsService.config && allowSubscriptions && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('subscriptions')))" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a>
<a *ngIf="postsService.config && enableDownloadsManager && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('downloads_manager')))" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a>
<ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && (!postsService.config.Advanced.multi_user_mode || (postsService.isLoggedIn && postsService.permissions.includes('subscriptions')))">
<a *ngIf="postsService.config && allowSubscriptions && postsService.hasPermission('subscriptions')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a>
<a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a>
<ng-container *ngIf="postsService.config && postsService.hasPermission('settings')">
<mat-divider></mat-divider>
<a mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/settings'><ng-container i18n="Settings menu label">Settings</ng-container></a>
</ng-container>
<ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
<mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
<a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatar [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatar>{{subscription.name}}</a>
</ng-container>
</mat-nav-list>

10
src/app/app.component.spec.ts
View File

@@ -1,9 +1,9 @@
import { TestBed, async } from '@angular/core/testing';
import { TestBed, waitForAsync } from '@angular/core/testing';
import { AppComponent } from './app.component';
describe('AppComponent', () => {
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [
AppComponent
@@ -11,19 +11,19 @@ describe('AppComponent', () => {
}).compileComponents();
}));
it('should create the app', async(() => {
it('should create the app', waitForAsync(() => {
const fixture = TestBed.createComponent(AppComponent);
const app = fixture.debugElement.componentInstance;
expect(app).toBeTruthy();
}));
it(`should have as title 'app'`, async(() => {
it(`should have as title 'app'`, waitForAsync(() => {
const fixture = TestBed.createComponent(AppComponent);
const app = fixture.debugElement.componentInstance;
expect(app.title).toEqual('app');
}));
it('should render title in a h1 tag', async(() => {
it('should render title in a h1 tag', waitForAsync(() => {
const fixture = TestBed.createComponent(AppComponent);
fixture.detectChanges();
const compiled = fixture.debugElement.nativeElement;

18
src/app/app.component.ts
View File

@@ -1,9 +1,6 @@
import { Component, OnInit, ElementRef, ViewChild, HostBinding, AfterViewInit } from '@angular/core';
import {MatDialogRef} from '@angular/material/dialog';
import {PostsService} from './posts.services';
import {FileCardComponent} from './file-card/file-card.component';
import { Observable } from 'rxjs/Observable';
import {FormControl, Validators} from '@angular/forms';
import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
import { MatDialog } from '@angular/material/dialog';
import { MatSidenav } from '@angular/material/sidenav';
import { MatSnackBar } from '@angular/material/snack-bar';
@@ -16,7 +13,6 @@ import 'rxjs/add/operator/filter'
import 'rxjs/add/operator/debounceTime'
import 'rxjs/add/operator/do'
import 'rxjs/add/operator/switch'
import { YoutubeSearchService, Result } from './youtube-search.service';
import { Router, NavigationStart, NavigationEnd } from '@angular/router';
import { OverlayContainer } from '@angular/cdk/overlay';
import { THEMES_CONFIG } from '../themes';
@@ -28,7 +24,11 @@ import { SetDefaultAdminDialogComponent } from './dialogs/set-default-admin-dial
@Component({
selector: 'app-root',
templateUrl: './app.component.html',
styleUrls: ['./app.component.css']
styleUrls: ['./app.component.css'],
providers: [{
provide: MatDialogRef,
useValue: {}
}]
})
export class AppComponent implements OnInit, AfterViewInit {
@@ -116,6 +116,12 @@ export class AppComponent implements OnInit, AfterViewInit {
if (this.allowSubscriptions) {
this.postsService.reloadSubscriptions();
}
this.postsService.reloadCategories();
this.postsService.getVersionInfo().subscribe(res => {
this.postsService.version_info = res['version_info'];
});
}
// theme stuff

31
src/app/app.module.ts
View File

@@ -30,16 +30,19 @@ import { MatSortModule } from '@angular/material/sort';
import { MatTableModule } from '@angular/material/table';
import { DragDropModule } from '@angular/cdk/drag-drop';
import { ClipboardModule } from '@angular/cdk/clipboard';
import { TextFieldModule } from '@angular/cdk/text-field';
import { FormsModule, ReactiveFormsModule } from '@angular/forms';
import { AppComponent } from './app.component';
import { HttpClientModule, HttpClient } from '@angular/common/http';
import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
import { PostsService } from 'app/posts.services';
import { FileCardComponent } from './file-card/file-card.component';
import { RouterModule } from '@angular/router';
import { AppRoutingModule } from './app-routing.module';
import { MainComponent } from './main/main.component';
import { PlayerComponent } from './player/player.component';
import { VgCoreModule, VgControlsModule, VgOverlayPlayModule, VgBufferingModule } from 'ngx-videogular';
import { VgControlsModule } from '@videogular/ngx-videogular/controls';
import { VgBufferingModule } from '@videogular/ngx-videogular/buffering';
import { VgOverlayPlayModule } from '@videogular/ngx-videogular/overlay-play';
import { VgCoreModule } from '@videogular/ngx-videogular/core';
import { InputDialogComponent } from './input-dialog/input-dialog.component';
import { LazyLoadImageModule, IsVisibleProps } from 'ng-lazyload-image';
import { audioFilesMouseHovering, videoFilesMouseHovering, audioFilesOpened, videoFilesOpened } from './main/main.component';
@@ -79,6 +82,12 @@ import { UnifiedFileCardComponent } from './components/unified-file-card/unified
import { RecentVideosComponent } from './components/recent-videos/recent-videos.component';
import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dialog/edit-subscription-dialog.component';
import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component';
import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component';
import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component';
import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.component';
import { H401Interceptor } from './http.interceptor';
import { ConcurrentStreamComponent } from './components/concurrent-stream/concurrent-stream.component';
import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component';
registerLocaleData(es, 'es');
@@ -89,7 +98,6 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
@NgModule({
declarations: [
AppComponent,
FileCardComponent,
MainComponent,
PlayerComponent,
InputDialogComponent,
@@ -105,6 +113,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
VideoInfoDialogComponent,
ArgModifierDialogComponent,
HighlightPipe,
LinkifyPipe,
UpdaterComponent,
UpdateProgressDialogComponent,
ShareMediaDialogComponent,
@@ -123,7 +132,12 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
UnifiedFileCardComponent,
RecentVideosComponent,
EditSubscriptionDialogComponent,
CustomPlaylistsComponent
CustomPlaylistsComponent,
EditCategoryDialogComponent,
TwitchChatComponent,
SeeMoreComponent,
ConcurrentStreamComponent,
SkipAdButtonComponent
],
imports: [
CommonModule,
@@ -162,6 +176,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
MatChipsModule,
DragDropModule,
ClipboardModule,
TextFieldModule,
NgxFileDropModule,
AvatarModule,
ContentLoaderModule,
@@ -181,10 +196,12 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
SettingsComponent
],
providers: [
PostsService
PostsService,
{ provide: HTTP_INTERCEPTORS, useClass: H401Interceptor, multi: true }
],
exports: [
HighlightPipe
HighlightPipe,
LinkifyPipe
],
bootstrap: [AppComponent]
})

6
src/app/components/concurrent-stream/concurrent-stream.component.html Normal file
View File

@@ -0,0 +1,6 @@
<div class="buttons-container">
<button (click)="startWatching()" *ngIf="!watch_together_clicked" mat-flat-button>Watch together</button>
<button (click)="startServer()" *ngIf="watch_together_clicked && !started && server_mode && server_already_exists === false" mat-flat-button>Start stream</button>
<button (click)="startClient()" *ngIf="watch_together_clicked && !started && server_already_exists === true" mat-flat-button>Join stream</button>
<button style="margin-left: 10px;" (click)="stop()" *ngIf="watch_together_clicked" mat-flat-button>Stop</button>
</div>

7
src/app/components/concurrent-stream/concurrent-stream.component.scss Normal file
View File

@@ -0,0 +1,7 @@
.buttons-container {
display: flex;
align-items: center;
justify-content: center;
margin-top: 15px;
margin-bottom: 15px;
}

25
src/app/components/concurrent-stream/concurrent-stream.component.spec.ts Normal file
View File

@@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { ConcurrentStreamComponent } from './concurrent-stream.component';
describe('ConcurrentStreamComponent', () => {
let component: ConcurrentStreamComponent;
let fixture: ComponentFixture<ConcurrentStreamComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ ConcurrentStreamComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(ConcurrentStreamComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

140
src/app/components/concurrent-stream/concurrent-stream.component.ts Normal file
View File

@@ -0,0 +1,140 @@
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-concurrent-stream',
templateUrl: './concurrent-stream.component.html',
styleUrls: ['./concurrent-stream.component.scss']
})
export class ConcurrentStreamComponent implements OnInit {
@Input() server_mode = false;
@Input() playback_timestamp;
@Input() playing;
@Input() uid;
@Output() setPlaybackTimestamp = new EventEmitter<any>();
@Output() togglePlayback = new EventEmitter<boolean>();
@Output() setPlaybackRate = new EventEmitter<number>();
started = false;
server_started = false;
watch_together_clicked = false;
server_already_exists = null;
check_timeout: any;
update_timeout: any;
PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_PLAYBACK_MODIFICATION = 0.5;
PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_SKIP = 2;
PLAYBACK_MODIFIER = 0.1;
playback_rate_modified = false;
constructor(private postsService: PostsService) { }
// flow: click start watching -> check for available stream to enable join button and if user, display "start stream"
// users who join a stream will send continuous requests for info on playback
ngOnInit(): void {
}
startServer() {
this.started = true;
this.server_started = true;
this.update_timeout = setInterval(() => {
this.updateStream();
}, 1000);
}
updateStream() {
this.postsService.updateConcurrentStream(this.uid, this.playback_timestamp, Date.now()/1000, this.playing).subscribe(res => {
});
}
startClient() {
this.started = true;
}
checkStream() {
if (this.server_started) { return; }
const current_playback_timestamp = this.playback_timestamp;
const current_unix_timestamp = Date.now()/1000;
this.postsService.checkConcurrentStream(this.uid).subscribe(res => {
const stream = res['stream'];
if (!stream) {
this.server_already_exists = false;
return;
}
this.server_already_exists = true;
// check whether client has joined the stream
if (!this.started) { return; }
if (!stream['playing'] && this.playing) {
// tell client to pause and set the timestamp to sync
this.togglePlayback.emit(false);
this.setPlaybackTimestamp.emit(stream['playback_timestamp']);
} else if (stream['playing']) {
// sync unpause state
if (!this.playing) { this.togglePlayback.emit(true); }
// sync time
const zeroed_local_unix_timestamp = current_unix_timestamp - current_playback_timestamp;
const zeroed_server_unix_timestamp = stream['unix_timestamp'] - stream['playback_timestamp'];
const seconds_behind_locally = zeroed_local_unix_timestamp - zeroed_server_unix_timestamp;
if (Math.abs(seconds_behind_locally) > this.PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_SKIP) {
// skip to playback timestamp because the difference is too high
this.setPlaybackTimestamp.emit(this.playback_timestamp + seconds_behind_locally + 0.3);
this.playback_rate_modified = false;
} else if (!this.playback_rate_modified && Math.abs(seconds_behind_locally) > this.PLAYBACK_TIMESTAMP_DIFFERENCE_THRESHOLD_PLAYBACK_MODIFICATION) {
// increase playback speed to avoid skipping
let seconds_to_wait = (Math.abs(seconds_behind_locally)/this.PLAYBACK_MODIFIER);
seconds_to_wait += 0.3/this.PLAYBACK_MODIFIER;
this.playback_rate_modified = true;
if (seconds_behind_locally > 0) {
// increase speed
this.setPlaybackRate.emit(1 + this.PLAYBACK_MODIFIER);
setTimeout(() => {
this.setPlaybackRate.emit(1);
this.playback_rate_modified = false;
}, seconds_to_wait * 1000);
} else {
// decrease speed
this.setPlaybackRate.emit(1 - this.PLAYBACK_MODIFIER);
setTimeout(() => {
this.setPlaybackRate.emit(1);
this.playback_rate_modified = false;
}, seconds_to_wait * 1000);
}
}
}
});
}
startWatching() {
this.watch_together_clicked = true;
this.check_timeout = setInterval(() => {
this.checkStream();
}, 1000);
}
stop() {
if (this.check_timeout) { clearInterval(this.check_timeout); }
if (this.update_timeout) { clearInterval(this.update_timeout); }
this.started = false;
this.server_started = false;
this.watch_together_clicked = false;
}
}

2
src/app/components/custom-playlists/custom-playlists.component.html
View File

@@ -2,7 +2,7 @@
<div class="container">
<div class="row justify-content-center">
<div *ngFor="let playlist of playlists; let i = index" class="mb-2 mt-2" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [loading]="false"></app-unified-file-card>
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToPlaylist($event)" [file_obj]="playlist" [is_playlist]="true" (editPlaylist)="editPlaylistDialog($event)" (deleteFile)="deletePlaylist($event)" [loading]="false"></app-unified-file-card>
</div>
</div>
</div>

4
src/app/components/custom-playlists/custom-playlists.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { CustomPlaylistsComponent } from './custom-playlists.component';
@@ -6,7 +6,7 @@ describe('CustomPlaylistsComponent', () => {
let component: CustomPlaylistsComponent;
let fixture: ComponentFixture<CustomPlaylistsComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ CustomPlaylistsComponent ]
})

31
src/app/components/custom-playlists/custom-playlists.component.ts
View File

@@ -24,11 +24,18 @@ export class CustomPlaylistsComponent implements OnInit {
this.getAllPlaylists();
}
});
this.postsService.playlists_changed.subscribe(changed => {
if (changed) {
this.getAllPlaylists();
}
});
}
getAllPlaylists() {
this.playlists_received = false;
this.postsService.getAllFiles().subscribe(res => {
// must call getAllFiles as we need to get category playlists as well
this.postsService.getPlaylists().subscribe(res => {
this.playlists = res['playlists'];
this.playlists_received = true;
});
@@ -53,16 +60,15 @@ export class CustomPlaylistsComponent implements OnInit {
goToPlaylist(info_obj) {
const playlist = info_obj.file;
const playlistID = playlist.id;
const type = playlist.type;
if (playlist) {
if (this.postsService.config['Extra']['download_only_mode']) {
this.downloading_content[type][playlistID] = true;
this.downloadPlaylist(playlist.fileNames, type, playlist.name, playlistID);
this.downloadPlaylist(playlist.id, playlist.name);
} else {
localStorage.setItem('player_navigator', this.router.url);
const fileNames = playlist.fileNames;
this.router.navigate(['/player', {fileNames: fileNames.join('|nvr|'), type: type, id: playlistID, uid: playlistID}]);
const routeParams = {playlist_id: playlistID};
if (playlist.auto) { routeParams['auto'] = playlist.auto; }
this.router.navigate(['/player', routeParams]);
}
} else {
// playlist not found
@@ -70,11 +76,12 @@ export class CustomPlaylistsComponent implements OnInit {
}
}
downloadPlaylist(fileNames, type, zipName = null, playlistID = null) {
this.postsService.downloadFileFromServer(fileNames, type, {outputName: zipName}).subscribe(res => {
if (playlistID) { this.downloading_content[type][playlistID] = false };
const blob: Blob = res;
saveAs(blob, zipName + '.zip');
downloadPlaylist(playlist_id, playlist_name) {
this.downloading_content[playlist_id] = true;
this.postsService.downloadPlaylistFromServer(playlist_id).subscribe(res => {
this.downloading_content[playlist_id] = false;
const blob: any = res;
saveAs(blob, playlist_name + '.zip');
});
}
@@ -97,7 +104,7 @@ export class CustomPlaylistsComponent implements OnInit {
const index = args.index;
const dialogRef = this.dialog.open(ModifyPlaylistComponent, {
data: {
playlist: playlist,
playlist_id: playlist.id,
width: '65vw'
}
});

114
src/app/components/downloads/downloads.component.html
View File

@@ -1,27 +1,91 @@
<div style="padding: 20px;">
<div *ngFor="let session_downloads of downloads | keyvalue">
<ng-container *ngIf="keys(session_downloads.value).length > 0">
<mat-card style="padding-bottom: 30px; margin-bottom: 15px;">
<h4 style="text-align: center;"><ng-container i18n="Session ID">Session ID:</ng-container>&nbsp;{{session_downloads.key}}
<span *ngIf="session_downloads.key === postsService.session_id">&nbsp;<ng-container i18n="Current session">(current)</ng-container></span>
</h4>
<div class="container">
<div class="row">
<div *ngFor="let download of session_downloads.value | keyvalue: sort_downloads; let i = index;" class="col-12 my-1">
<mat-card *ngIf="download.value" class="mat-elevation-z3">
<app-download-item [download]="download.value" [queueNumber]="i+1" (cancelDownload)="clearDownload(session_downloads.key, download.value.uid)"></app-download-item>
</mat-card>
</div>
</div>
</div>
<div>
<button style="top: 15px;" (click)="clearDownloads(session_downloads.key)" mat-stroked-button color="warn"><ng-container i18n="clear all downloads action button">Clear all downloads</ng-container></button>
</div>
</mat-card>
</ng-container>
</div>
<div [hidden]="!(downloads && downloads.length > 0)">
<div style="overflow: hidden;" [ngClass]="uids ? 'rounded mat-elevation-z2' : 'mat-elevation-z8'">
<mat-table style="overflow: hidden" [ngClass]="uids ? 'rounded-top' : null" matSort [dataSource]="dataSource">
<!-- Date Column -->
<ng-container matColumnDef="date">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Date">Date</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element"> {{element.timestamp_start | date: 'short'}} </mat-cell>
</ng-container>
<!-- Title Column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<span class="one-line" [matTooltip]="element.title ? element.title : null">
{{element.title}}
</span>
</mat-cell>
</ng-container>
<div *ngIf="downloads && !downloadsValid()">
<h4 style="text-align: center;" i18n="No downloads label">No downloads available!</h4>
</div>
<!-- Subscription Column -->
<ng-container matColumnDef="subscription">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Subscription">Subscription</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<ng-container *ngIf="element.sub_name">
{{element.sub_name}}
</ng-container>
<ng-container *ngIf="!element.sub_name">
N/A
</ng-container>
</mat-cell>
</ng-container>
<!-- Stage Column -->
<ng-container matColumnDef="stage">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Stage">Stage</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element"> {{STEP_INDEX_TO_LABEL[element.step_index]}} </mat-cell>
</ng-container>
<!-- Progress Column -->
<ng-container matColumnDef="progress">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Progress">Progress</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<ng-container *ngIf="element.percent_complete">
{{+(element.percent_complete) > 100 ? '100' : element.percent_complete}}%
</ng-container>
<ng-container *ngIf="!element.percent_complete">
N/A
</ng-container>
</mat-cell>
</ng-container>
<!-- Actions Column -->
<ng-container matColumnDef="actions">
<mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<div>
<ng-container *ngIf="!element.finished">
<button (click)="pauseDownload(element.uid)" *ngIf="!element.paused || !element.finished_step" [disabled]="element.paused && !element.finished_step" mat-icon-button matTooltip="Pause" i18n-matTooltip="Pause"><mat-spinner [diameter]="28" *ngIf="element.paused && !element.finished_step" class="icon-button-spinner"></mat-spinner><mat-icon>pause</mat-icon></button>
<button (click)="resumeDownload(element.uid)" *ngIf="element.paused && element.finished_step" mat-icon-button matTooltip="Resume" i18n-matTooltip="Resume"><mat-icon>play_arrow</mat-icon></button>
<button *ngIf="false && !element.paused" (click)="cancelDownload(element.uid)" mat-icon-button matTooltip="Cancel" i18n-matTooltip="Cancel"><mat-icon>cancel</mat-icon></button>
</ng-container>
<ng-container *ngIf="element.finished">
<button *ngIf="!element.error" (click)="watchContent(element)" mat-icon-button matTooltip="Watch content" i18n-matTooltip="Watch content"><mat-icon>smart_display</mat-icon></button>
<button *ngIf="element.error" (click)="showError(element)" mat-icon-button matTooltip="Show error" i18n-matTooltip="Show error"><mat-icon>warning</mat-icon></button>
<button (click)="restartDownload(element.uid)" mat-icon-button matTooltip="Restart" i18n-matTooltip="Restart"><mat-icon>restart_alt</mat-icon></button>
</ng-container>
<button *ngIf="element.finished || element.paused" (click)="clearDownload(element.uid)" mat-icon-button matTooltip="Clear" i18n-matTooltip="Clear"><mat-icon>delete</mat-icon></button>
</div>
</mat-cell>
</ng-container>
<mat-header-row [ngClass]="uids ? 'rounded-top' : null" *matHeaderRowDef="displayedColumns"></mat-header-row>
<mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
</mat-table>
<mat-paginator [ngClass]="uids ? 'rounded-bottom' : null" [pageSizeOptions]="[5, 10, 20]"
showFirstLastButtons
aria-label="Select page of downloads">
</mat-paginator>
</div>
<div *ngIf="!uids" class="downloads-action-button-div">
<button [disabled]="!running_download_exists" mat-stroked-button (click)="pauseAllDownloads()"><ng-container i18n="Pause all downloads">Pause all downloads</ng-container></button>
<button style="margin-left: 10px;" [disabled]="!paused_download_exists" mat-stroked-button (click)="resumeAllDownloads()"><ng-container i18n="Resume all downloads">Resume all downloads</ng-container></button>
<button color="warn" style="margin-left: 10px;" mat-stroked-button (click)="clearFinishedDownloads()"><ng-container i18n="Clear finished downloads">Clear finished downloads</ng-container></button>
</div>
</div>
<div *ngIf="(!downloads || downloads.length === 0) && downloads_retrieved && !uids">
<h4 style="text-align: center; margin-top: 10px;" i18n="No downloads label">No downloads available!</h4>
</div>

32
src/app/components/downloads/downloads.component.scss
View File

@@ -0,0 +1,32 @@
mat-header-cell, mat-cell {
justify-content: center;
}
.one-line {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.icon-button-spinner {
position: absolute;
top: 7px;
left: 6px;
}
.downloads-action-button-div {
margin-top: 10px;
margin-left: 5px;
}
.rounded-top {
border-radius: 16px 16px 0px 0px !important;
}
.rounded-bottom {
border-radius: 0px 0px 16px 16px !important;
}
.rounded {
border-radius: 16px 16px 16px 16px !important;
}

4
src/app/components/downloads/downloads.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { DownloadsComponent } from './downloads.component';
@@ -6,7 +6,7 @@ describe('DownloadsComponent', () => {
let component: DownloadsComponent;
let fixture: ComponentFixture<DownloadsComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ DownloadsComponent ]
})

286
src/app/components/downloads/downloads.component.ts
View File

@@ -1,7 +1,13 @@
import { Component, OnInit, ViewChildren, QueryList, ElementRef, OnDestroy } from '@angular/core';
import { Component, OnInit, OnDestroy, ViewChild, Input, EventEmitter } from '@angular/core';
import { PostsService } from 'app/posts.services';
import { trigger, transition, animateChild, stagger, query, style, animate } from '@angular/animations';
import { Router } from '@angular/router';
import { MatPaginator } from '@angular/material/paginator';
import { MatTableDataSource } from '@angular/material/table';
import { MatDialog } from '@angular/material/dialog';
import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
import { MatSort } from '@angular/material/sort';
import { Clipboard } from '@angular/cdk/clipboard';
@Component({
selector: 'app-downloads',
@@ -34,138 +40,222 @@ import { Router } from '@angular/router';
})
export class DownloadsComponent implements OnInit, OnDestroy {
@Input() uids = null;
downloads_check_interval = 1000;
downloads = {};
downloads = [];
finished_downloads = [];
interval_id = null;
keys = Object.keys;
valid_sessions_length = 0;
paused_download_exists = false;
running_download_exists = false;
STEP_INDEX_TO_LABEL = {
0: $localize`Creating download`,
1: $localize`Getting info`,
2: $localize`Downloading file`,
3: $localize`Complete`
}
displayedColumns: string[] = ['date', 'title', 'stage', 'subscription', 'progress', 'actions'];
dataSource = null; // new MatTableDataSource<Download>();
downloads_retrieved = false;
@ViewChild(MatPaginator) paginator: MatPaginator;
@ViewChild(MatSort) sort: MatSort;
sort_downloads = (a, b) => {
const result = b.value.timestamp_start - a.value.timestamp_start;
const result = b.timestamp_start - a.timestamp_start;
return result;
}
constructor(public postsService: PostsService, private router: Router) { }
constructor(public postsService: PostsService, private router: Router, private dialog: MatDialog, private clipboard: Clipboard) { }
ngOnInit(): void {
if (this.postsService.initialized) {
this.getCurrentDownloadsRecurring();
} else {
this.postsService.service_initialized.subscribe(init => {
if (init) {
this.getCurrentDownloadsRecurring();
}
});
}
}
getCurrentDownloadsRecurring(): void {
if (!this.postsService.config['Extra']['enable_downloads_manager']) {
this.router.navigate(['/home']);
return;
}
this.getCurrentDownloads();
this.interval_id = setInterval(() => {
this.getCurrentDownloads();
}, this.downloads_check_interval);
this.postsService.service_initialized.subscribe(init => {
if (init) {
if (!this.postsService.config['Extra']['enable_downloads_manager']) {
this.router.navigate(['/home']);
}
}
});
}
ngOnDestroy() {
ngOnDestroy(): void {
if (this.interval_id) { clearInterval(this.interval_id) }
}
getCurrentDownloads() {
this.postsService.getCurrentDownloads().subscribe(res => {
if (res['downloads']) {
this.assignNewValues(res['downloads']);
getCurrentDownloads(): void {
this.postsService.getCurrentDownloads(this.uids).subscribe(res => {
this.downloads_retrieved = true;
if (res['downloads'] !== null
&& res['downloads'] !== undefined
&& JSON.stringify(this.downloads) !== JSON.stringify(res['downloads'])) {
this.downloads = this.combineDownloads(this.downloads, res['downloads']);
// this.downloads = res['downloads'];
this.downloads.sort(this.sort_downloads);
this.dataSource = new MatTableDataSource<Download>(this.downloads);
this.dataSource.paginator = this.paginator;
this.dataSource.sort = this.sort;
this.paused_download_exists = this.downloads.find(download => download['paused'] && !download['error']);
this.running_download_exists = this.downloads.find(download => !download['paused'] && !download['finished']);
} else {
// failed to get downloads
}
});
}
clearDownload(session_id, download_uid) {
this.postsService.clearDownloads(false, session_id, download_uid).subscribe(res => {
if (res['success']) {
// this.downloads = res['downloads'];
} else {
clearFinishedDownloads(): void {
const dialogRef = this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: $localize`Clear finished downloads`,
dialogText: $localize`Would you like to clear your finished downloads?`,
submitText: $localize`Clear`,
warnSubmitColor: true
}
});
}
clearDownloads(session_id) {
this.postsService.clearDownloads(false, session_id).subscribe(res => {
if (res['success']) {
this.downloads = res['downloads'];
} else {
}
});
}
clearAllDownloads() {
this.postsService.clearDownloads(true).subscribe(res => {
if (res['success']) {
this.downloads = res['downloads'];
} else {
}
});
}
assignNewValues(new_downloads_by_session) {
const session_keys = Object.keys(new_downloads_by_session);
// remove missing session IDs
const current_session_ids = Object.keys(this.downloads);
const missing_session_ids = current_session_ids.filter(session => session_keys.indexOf(session) === -1)
for (const missing_session_id of missing_session_ids) {
delete this.downloads[missing_session_id];
}
// loop through sessions
for (let i = 0; i < session_keys.length; i++) {
const session_id = session_keys[i];
const session_downloads_by_id = new_downloads_by_session[session_id];
const session_download_ids = Object.keys(session_downloads_by_id);
if (this.downloads[session_id]) {
// remove missing download IDs
const current_download_ids = Object.keys(this.downloads[session_id]);
const missing_download_ids = current_download_ids.filter(download => session_download_ids.indexOf(download) === -1)
for (const missing_download_id of missing_download_ids) {
console.log('removing missing download id');
delete this.downloads[session_id][missing_download_id];
}
}
if (!this.downloads[session_id]) {
this.downloads[session_id] = session_downloads_by_id;
} else {
for (let j = 0; j < session_download_ids.length; j++) {
const download_id = session_download_ids[j];
const download = new_downloads_by_session[session_id][download_id]
if (!this.downloads[session_id][download_id]) {
this.downloads[session_id][download_id] = download;
} else {
const download_to_update = this.downloads[session_id][download_id];
download_to_update['percent_complete'] = download['percent_complete'];
download_to_update['complete'] = download['complete'];
download_to_update['timestamp_end'] = download['timestamp_end'];
download_to_update['downloading'] = download['downloading'];
download_to_update['error'] = download['error'];
dialogRef.afterClosed().subscribe(confirmed => {
if (confirmed) {
this.postsService.clearFinishedDownloads().subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to clear finished downloads!');
}
}
});
}
});
}
pauseDownload(download_uid: string): void {
this.postsService.pauseDownload(download_uid).subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
}
});
}
pauseAllDownloads(): void {
this.postsService.pauseAllDownloads().subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to pause all downloads! See server logs for more info.');
}
});
}
resumeDownload(download_uid: string): void {
this.postsService.resumeDownload(download_uid).subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to resume download! See server logs for more info.');
}
});
}
resumeAllDownloads(): void {
this.postsService.resumeAllDownloads().subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to resume all downloads! See server logs for more info.');
}
});
}
restartDownload(download_uid: string): void {
this.postsService.restartDownload(download_uid).subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to restart download! See server logs for more info.');
}
});
}
cancelDownload(download_uid: string): void {
this.postsService.cancelDownload(download_uid).subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to cancel download! See server logs for more info.');
}
});
}
clearDownload(download_uid: string): void {
this.postsService.clearDownload(download_uid).subscribe(res => {
if (!res['success']) {
this.postsService.openSnackBar('Failed to pause download! See server logs for more info.');
}
});
}
watchContent(download): void {
const container = download['container'];
localStorage.setItem('player_navigator', this.router.url.split(';')[0]);
const is_playlist = container['uids']; // hacky, TODO: fix
if (is_playlist) {
this.router.navigate(['/player', {playlist_id: container['id'], type: download['type']}]);
} else {
this.router.navigate(['/player', {type: download['type'], uid: container['uid']}]);
}
}
downloadsValid() {
let valid = false;
const keys = this.keys(this.downloads);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const value = this.downloads[key];
if (this.keys(value).length > 0) {
valid = true;
break;
}
}
return valid;
combineDownloads(downloads_old, downloads_new) {
// only keeps downloads that exist in the new set
downloads_old = downloads_old.filter(download_old => downloads_new.some(download_new => download_new.uid === download_old.uid));
// add downloads from the new set that the old one doesn't have
const downloads_to_add = downloads_new.filter(download_new => !downloads_old.some(download_old => download_new.uid === download_old.uid));
downloads_old.push(...downloads_to_add);
downloads_old.forEach(download_old => {
const download_new = downloads_new.find(download_to_check => download_old.uid === download_to_check.uid);
Object.keys(download_new).forEach(key => {
download_old[key] = download_new[key];
});
Object.keys(download_old).forEach(key => {
if (!download_new[key]) delete download_old[key];
});
});
return downloads_old;
}
showError(download) {
const copyToClipboardEmitter = new EventEmitter<boolean>();
this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: $localize`Error for ${download['url']}:url:`,
dialogText: download['error'],
submitText: $localize`Copy to clipboard`,
cancelText: $localize`Close`,
closeOnSubmit: false,
onlyEmitOnDone: true,
doneEmitter: copyToClipboardEmitter
}
});
copyToClipboardEmitter.subscribe(done => {
if (done) {
this.postsService.openSnackBar($localize`Copied to clipboard!`);
this.clipboard.copy(download['error']);
}
});
}
}
export interface Download {
timestamp_start: number;
title: string;
step_index: number;
progress: string;
}

16
src/app/components/login/login.component.html
View File

@@ -1,5 +1,5 @@
<mat-card class="login-card">
<mat-tab-group [(selectedIndex)]="selectedTabIndex">
<mat-tab-group style="margin-bottom: 20px" [(selectedIndex)]="selectedTabIndex">
<mat-tab label="Login">
<div style="margin-top: 10px;">
<mat-form-field>
@@ -11,9 +11,6 @@
<input [(ngModel)]="loginPasswordInput" (keyup.enter)="login()" type="password" matInput placeholder="Password">
</mat-form-field>
</div>
<div style="margin-bottom: 10px; margin-top: 10px;">
<button [disabled]="loggingIn" color="primary" (click)="login()" mat-raised-button><ng-container i18n="Login">Login</ng-container></button>
</div>
</mat-tab>
<mat-tab *ngIf="registrationEnabled" label="Register">
<div style="margin-top: 10px;">
@@ -31,9 +28,14 @@
<input [(ngModel)]="registrationPasswordConfirmationInput" type="password" matInput placeholder="Confirm Password">
</mat-form-field>
</div>
<div style="margin-bottom: 10px; margin-top: 10px;">
<button [disabled]="registering" color="primary" (click)="register()" mat-raised-button><ng-container i18n="Register">Register</ng-container></button>
</div>
</mat-tab>
</mat-tab-group>
<div *ngIf="selectedTabIndex === 0" class="login-button-div">
<button [disabled]="loggingIn" color="primary" (click)="login()" mat-raised-button><ng-container i18n="Login">Login</ng-container></button>
<mat-progress-bar *ngIf="loggingIn" class="login-progress-bar" mode="indeterminate"></mat-progress-bar>
</div>
<div *ngIf="selectedTabIndex === 1" class="login-button-div">
<button [disabled]="registering" color="primary" (click)="register()" mat-raised-button><ng-container i18n="Register">Register</ng-container></button>
<mat-progress-bar *ngIf="registering" class="login-progress-bar" mode="indeterminate"></mat-progress-bar>
</div>
</mat-card>

29
src/app/components/login/login.component.scss
View File

@@ -1,6 +1,33 @@
.login-card {
max-width: 600px;
max-width: 400px;
width: 80%;
margin: 0 auto;
margin-top: 20px;
padding-top: 8px;
}
.login-div {
height: calc(100% - 170px);
overflow-y: auto;
}
.login-button-div {
margin-bottom: 10px;
margin-top: 10px;
margin-left: -16px;
margin-right: -16px;
bottom: 0px;
width: 100%;
position: absolute;
}
.login-button-div > button {
width: 100%;
border-radius: 0px 0px 4px 4px !important;
}
.login-progress-bar {
position: absolute;
bottom: 0px;
border-radius: 0px 0px 4px 4px;
}

4
src/app/components/login/login.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { LoginComponent } from './login.component';
@@ -6,7 +6,7 @@ describe('LoginComponent', () => {
let component: LoginComponent;
let fixture: ComponentFixture<LoginComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ LoginComponent ]
})

2
src/app/components/login/login.component.ts
View File

@@ -27,7 +27,7 @@ export class LoginComponent implements OnInit {
constructor(private postsService: PostsService, private snackBar: MatSnackBar, private router: Router) { }
ngOnInit(): void {
if (this.postsService.isLoggedIn) {
if (this.postsService.isLoggedIn && localStorage.getItem('jwt_token') !== 'null') {
this.router.navigate(['/home']);
}
this.postsService.service_initialized.subscribe(init => {

4
src/app/components/logs-viewer/logs-viewer.component.html
View File

@@ -1,4 +1,4 @@
<div style="height: 275px;">
<div style="height: 100%;">
<div *ngIf="logs_loading" style="z-index: 999; position: absolute; top: 40%; left: 50%">
<mat-spinner [diameter]="32"></mat-spinner>
</div>
@@ -10,7 +10,7 @@
</cdk-virtual-scroll-viewport>-->
<!-- Non-virtual mode (slow, bug-free) -->
<div style="height: 274px; overflow-y: auto">
<div style="height: 100%; overflow-y: auto">
<div *ngFor="let log of logs; let i = index" class="example-item">
<span [ngStyle]="{'color':log.color}">{{log.text}}</span>
</div>

4
src/app/components/logs-viewer/logs-viewer.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { LogsViewerComponent } from './logs-viewer.component';
@@ -6,7 +6,7 @@ describe('LogsViewerComponent', () => {
let component: LogsViewerComponent;
let fixture: ComponentFixture<LogsViewerComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ LogsViewerComponent ]
})

3
src/app/components/logs-viewer/logs-viewer.component.ts
View File

@@ -61,7 +61,8 @@ export class LogsViewerComponent implements OnInit {
data: {
dialogTitle: 'Clear logs',
dialogText: 'Would you like to clear your logs? This will delete all your current logs, permanently.',
submitText: 'Clear'
submitText: 'Clear',
warnSubmitColor: true
}
});
dialogRef.afterClosed().subscribe(confirmed => {

2
src/app/components/manage-role/manage-role.component.html
View File

@@ -5,7 +5,7 @@
<mat-list-item role="listitem" *ngFor="let permission of available_permissions">
<h3 matLine>{{permissionToLabel[permission] ? permissionToLabel[permission] : permission}}</h3>
<span matLine>
<mat-radio-group [disabled]="permission === 'settings' && role.name === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
<mat-radio-group [disabled]="permission === 'settings' && role.key === 'admin'" (change)="changeRolePermissions($event, permission, permissions[permission])" [(ngModel)]="permissions[permission]" [attr.aria-label]="'Give role permission for ' + permission">
<mat-radio-button value="yes"><ng-container i18n="Yes">Yes</ng-container></mat-radio-button>
<mat-radio-button value="no"><ng-container i18n="No">No</ng-container></mat-radio-button>
</mat-radio-group>

4
src/app/components/manage-role/manage-role.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { ManageRoleComponent } from './manage-role.component';
@@ -6,7 +6,7 @@ describe('ManageRoleComponent', () => {
let component: ManageRoleComponent;
let fixture: ComponentFixture<ManageRoleComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ ManageRoleComponent ]
})

2
src/app/components/manage-role/manage-role.component.ts
View File

@@ -47,7 +47,7 @@ export class ManageRoleComponent implements OnInit {
}
changeRolePermissions(change, permission) {
this.postsService.setRolePermission(this.role.name, permission, change.value).subscribe(res => {
this.postsService.setRolePermission(this.role.key, permission, change.value).subscribe(res => {
if (res['success']) {
} else {

4
src/app/components/manage-user/manage-user.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { ManageUserComponent } from './manage-user.component';
@@ -6,7 +6,7 @@ describe('ManageUserComponent', () => {
let component: ManageUserComponent;
let fixture: ComponentFixture<ManageUserComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ ManageUserComponent ]
})

4
src/app/components/modify-users/modify-users.component.html
View File

@@ -1,7 +1,7 @@
<div *ngIf="dataSource; else loading">
<div style="padding: 15px">
<div class="row">
<div class="table table-responsive px-5 pb-4 pt-2">
<div class="table table-responsive pb-4 pt-2">
<div class="example-header">
<mat-form-field>
<input matInput (keyup)="applyFilter($event.target.value)" placeholder="Search" i18n-placeholder="search field description">
@@ -94,7 +94,7 @@
</div>
<button color="primary" [matMenuTriggerFor]="edit_roles_menu" class="edit-role" mat-raised-button><ng-container i18n="Edit role">Edit Role</ng-container></button>
<mat-menu #edit_roles_menu="matMenu">
<button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.name}}</button>
<button (click)="openModifyRole(role)" mat-menu-item *ngFor="let role of roles">{{role.key}}</button>
</mat-menu>
</div>

1
src/app/components/modify-users/modify-users.component.scss
View File

@@ -1,5 +1,4 @@
.edit-role {
position: relative;
top: -50px;
left: 35px;
}

4
src/app/components/modify-users/modify-users.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { ModifyUsersComponent } from './modify-users.component';
@@ -6,7 +6,7 @@ describe('ModifyUsersComponent', () => {
let component: ModifyUsersComponent;
let fixture: ComponentFixture<ModifyUsersComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ ModifyUsersComponent ]
})

11
src/app/components/modify-users/modify-users.component.ts
View File

@@ -78,16 +78,7 @@ export class ModifyUsersComponent implements OnInit, AfterViewInit {
getRoles() {
this.postsService.getRoles().subscribe(res => {
this.roles = [];
const roles = res['roles'];
const role_names = Object.keys(roles);
for (let i = 0; i < role_names.length; i++) {
const role_name = role_names[i];
this.roles.push({
name: role_name,
permissions: roles[role_name]['permissions']
});
}
this.roles = res['roles'];
});
}

30
src/app/components/recent-videos/recent-videos.component.html
View File

@@ -28,18 +28,38 @@
</div>
</div>
<div>
<div class="container">
<div class="container" style="margin-bottom: 16px">
<div class="row justify-content-center">
<ng-container *ngIf="normal_files_received">
<div *ngFor="let file of filtered_files; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [loading]="false" (deleteFile)="deleteFile($event)"></app-unified-file-card>
<ng-container *ngIf="normal_files_received && paged_data">
<div *ngFor="let file of paged_data; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [availablePlaylists]="playlists" (addFileToPlaylist)="addFileToPlaylist($event)" [loading]="false" (deleteFile)="deleteFile($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? '?jwt=' + this.postsService.token : ''"></app-unified-file-card>
</div>
<div *ngIf="paged_data.length === 0">
<ng-container i18n="No videos found">No videos found.</ng-container>
</div>
</ng-container>
<ng-container *ngIf="!normal_files_received && loading_files && loading_files.length > 0">
<div *ngFor="let file of loading_files; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" [loading]="true" [theme]="postsService.theme"></app-unified-file-card>
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" [loading]="true" [theme]="postsService.theme"></app-unified-file-card>
</div>
</ng-container>
</div>
</div>
<div>
<div style="position: absolute; margin-left: -8px; margin-top: 5px; scale: 0.8">
<mat-form-field>
<mat-label><ng-container i18n="File type">File type</ng-container></mat-label>
<mat-select color="accent" [(ngModel)]="fileTypeFilter" (selectionChange)="fileTypeFilterChanged($event.value)">
<mat-option value="both"><ng-container i18n="Both">Both</ng-container></mat-option>
<mat-option value="video_only"><ng-container i18n="Video only">Video only</ng-container></mat-option>
<mat-option value="audio_only"><ng-container i18n="Audio only">Audio only</ng-container></mat-option>
</mat-select>
</mat-form-field>
</div>
<mat-paginator class="paginator" #paginator *ngIf="paged_data && paged_data.length > 0" (page)="pageChangeEvent($event)" [length]="file_count"
[pageSize]="pageSize"
[pageSizeOptions]="[5, 10, 25, 100, this.paged_data && this.paged_data.length > 100 ? this.paged_data.length : 250]">
</mat-paginator>
</div>
</div>

4
src/app/components/recent-videos/recent-videos.component.scss
View File

@@ -47,6 +47,10 @@
top: 10px;
}
.paginator {
margin-top: 5px;
}
.my-videos-title {
text-align: center;
position: relative;

4
src/app/components/recent-videos/recent-videos.component.spec.ts
View File

@@ -1,4 +1,4 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
import { RecentVideosComponent } from './recent-videos.component';
@@ -6,7 +6,7 @@ describe('RecentVideosComponent', () => {
let component: RecentVideosComponent;
let fixture: ComponentFixture<RecentVideosComponent>;
beforeEach(async(() => {
beforeEach(waitForAsync(() => {
TestBed.configureTestingModule({
declarations: [ RecentVideosComponent ]
})

211
src/app/components/recent-videos/recent-videos.component.ts
View File

@@ -1,7 +1,10 @@
import { Component, OnInit } from '@angular/core';
import { Component, OnInit, ViewChild } from '@angular/core';
import { PostsService } from 'app/posts.services';
import { Router } from '@angular/router';
import { FileType } from '../../../api-types';
import { MatPaginator } from '@angular/material/paginator';
import { Subject } from 'rxjs';
import { distinctUntilChanged } from 'rxjs/operators';
@Component({
selector: 'app-recent-videos',
@@ -15,8 +18,8 @@ export class RecentVideosComponent implements OnInit {
normal_files_received = false;
subscription_files_received = false;
files: any[] = null;
filtered_files: any[] = null;
file_count = 10;
searchChangedSubject: Subject<string> = new Subject<string>();
downloading_content = {'video': {}, 'audio': {}};
search_mode = false;
search_text = '';
@@ -50,11 +53,20 @@ export class RecentVideosComponent implements OnInit {
}
};
filterProperty = this.filterProperties['upload_date'];
fileTypeFilter = 'both';
playlists = null;
pageSize = 10;
paged_data = null;
@ViewChild('paginator') paginator: MatPaginator
constructor(public postsService: PostsService, private router: Router) {
// get cached file count
if (localStorage.getItem('cached_file_count')) {
this.cached_file_count = +localStorage.getItem('cached_file_count');
this.cached_file_count = +localStorage.getItem('cached_file_count') <= 10 ? +localStorage.getItem('cached_file_count') : 10;
this.loading_files = Array(this.cached_file_count).fill(0);
}
}
@@ -62,78 +74,108 @@ export class RecentVideosComponent implements OnInit {
ngOnInit(): void {
if (this.postsService.initialized) {
this.getAllFiles();
this.getAllPlaylists();
}
this.postsService.service_initialized.subscribe(init => {
if (init) {
this.getAllFiles();
this.getAllPlaylists();
}
});
this.postsService.files_changed.subscribe(changed => {
if (changed) {
this.getAllFiles();
}
});
// set filter property to cached
this.postsService.playlists_changed.subscribe(changed => {
if (changed) {
this.getAllPlaylists();
}
});
// set filter property to cached value
const cached_filter_property = localStorage.getItem('filter_property');
if (cached_filter_property && this.filterProperties[cached_filter_property]) {
this.filterProperty = this.filterProperties[cached_filter_property];
}
// set file type filter to cached value
const cached_file_type_filter = localStorage.getItem('file_type_filter');
if (cached_file_type_filter) {
this.fileTypeFilter = cached_file_type_filter;
}
const sort_order = localStorage.getItem('recent_videos_sort_order');
if (sort_order) {
this.descendingMode = sort_order === 'descending';
}
this.searchChangedSubject
.debounceTime(500)
.pipe(distinctUntilChanged()
).subscribe(model => {
if (model.length > 0) {
this.search_mode = true;
} else {
this.search_mode = false;
}
this.getAllFiles();
});
}
getAllPlaylists() {
this.postsService.getPlaylists().subscribe(res => {
this.playlists = res['playlists'];
});
}
// search
onSearchInputChanged(newvalue) {
if (newvalue.length > 0) {
this.search_mode = true;
this.filterFiles(newvalue);
} else {
this.search_mode = false;
this.filtered_files = this.files;
}
}
private filterFiles(value: string) {
const filterValue = value.toLowerCase();
this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue));
}
filterByProperty(prop) {
if (this.descendingMode) {
this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? -1 : 1));
} else {
this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? 1 : -1));
}
this.normal_files_received = false;
this.searchChangedSubject.next(newvalue);
}
filterOptionChanged(value) {
this.filterByProperty(value['property']);
localStorage.setItem('filter_property', value['key']);
this.getAllFiles();
}
fileTypeFilterChanged(value) {
localStorage.setItem('file_type_filter', value);
this.getAllFiles();
}
toggleModeChange() {
this.descendingMode = !this.descendingMode;
this.filterByProperty(this.filterProperty['property']);
localStorage.setItem('recent_videos_sort_order', this.descendingMode ? 'descending' : 'ascending');
this.getAllFiles();
}
// get files
getAllFiles() {
this.normal_files_received = false;
this.postsService.getAllFiles().subscribe(res => {
this.files = res['files'];
this.files.forEach(file => {
getAllFiles(cache_mode = false) {
this.normal_files_received = cache_mode;
const current_file_index = (this.paginator?.pageIndex ? this.paginator.pageIndex : 0)*this.pageSize;
const sort = {by: this.filterProperty['property'], order: this.descendingMode ? -1 : 1};
const range = [current_file_index, current_file_index + this.pageSize];
this.postsService.getAllFiles(sort, range, this.search_mode ? this.search_text : null, this.fileTypeFilter).subscribe(res => {
this.file_count = res['file_count'];
this.paged_data = res['files'];
for (let i = 0; i < this.paged_data.length; i++) {
const file = this.paged_data[i];
file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration);
});
this.files.sort(this.sortFiles);
if (this.search_mode) {
this.filterFiles(this.search_text);
} else {
this.filtered_files = this.files;
}
this.filterByProperty(this.filterProperty['property']);
// set cached file count for future use, note that we convert the amount of files to a string
localStorage.setItem('cached_file_count', '' + this.files.length);
localStorage.setItem('cached_file_count', '' + this.file_count);
this.normal_files_received = true;
});
}
@@ -155,15 +197,14 @@ export class RecentVideosComponent implements OnInit {
const sub = this.postsService.getSubscriptionByID(file.sub_id);
if (sub.streamingOnly) {
// streaming only mode subscriptions
!new_tab ? this.router.navigate(['/player', {name: file.id,
url: file.requested_formats ? file.requested_formats[0].url : file.url}])
: window.open(`/#/player;name=${file.id};url=${file.requested_formats ? file.requested_formats[0].url : file.url}`);
// !new_tab ? this.router.navigate(['/player', {name: file.id,
// url: file.requested_formats ? file.requested_formats[0].url : file.url}])
// : window.open(`/#/player;name=${file.id};url=${file.requested_formats ? file.requested_formats[0].url : file.url}`);
} else {
// normal subscriptions
!new_tab ? this.router.navigate(['/player', {fileNames: file.id,
type: file.isAudio ? 'audio' : 'video', subscriptionName: sub.name,
subPlaylist: sub.isPlaylist}])
: window.open(`/#/player;fileNames=${file.id};type=${file.isAudio ? 'audio' : 'video'};subscriptionName=${sub.name};subPlaylist=${sub.isPlaylist}`);
!new_tab ? this.router.navigate(['/player', {uid: file.uid,
type: file.isAudio ? 'audio' : 'video'}])
: window.open(`/#/player;uid=${file.uid};type=${file.isAudio ? 'audio' : 'video'}`);
}
} else {
// normal files
@@ -190,15 +231,12 @@ export class RecentVideosComponent implements OnInit {
const type = (file.isAudio ? 'audio' : 'video') as FileType;
const ext = type === 'audio' ? '.mp3' : '.mp4'
const sub = this.postsService.getSubscriptionByID(file.sub_id);
this.postsService.downloadFileFromServer(
file.id, type,
{subscriptionName: sub.name, subPlaylist: sub.isPlaylist, uid: this.postsService.user ? this.postsService.user.uid : null}
).subscribe(res => {
const blob: Blob = res;
saveAs(blob, file.id + ext);
}, err => {
console.log(err);
});
this.postsService.downloadFileFromServer(file.uid).subscribe(res => {
const blob: Blob = res;
saveAs(blob, file.id + ext);
}, err => {
console.log(err);
});
}
downloadNormalFile(file) {
@@ -206,14 +244,14 @@ export class RecentVideosComponent implements OnInit {
const ext = type === 'audio' ? '.mp3' : '.mp4'
const name = file.id;
this.downloading_content[type][name] = true;
this.postsService.downloadFileFromServer(name, type).subscribe(res => {
this.postsService.downloadFileFromServer(file.uid).subscribe(res => {
this.downloading_content[type][name] = false;
const blob: Blob = res;
saveAs(blob, decodeURIComponent(name) + ext);
if (!this.postsService.config.Extra.file_manager_enabled) {
// tell server to delete the file once downloaded
this.postsService.deleteFile(name, false).subscribe(delRes => {
this.postsService.deleteFile(file.uid).subscribe(delRes => {
// reload mp4s
this.getAllFiles();
});
@@ -229,17 +267,17 @@ export class RecentVideosComponent implements OnInit {
const blacklistMode = args.blacklistMode;
if (file.sub_id) {
this.deleteSubscriptionFile(file, index, blacklistMode);
this.deleteSubscriptionFile(file, blacklistMode);
} else {
this.deleteNormalFile(file, index, blacklistMode);
this.deleteNormalFile(file, blacklistMode);
}
}
deleteNormalFile(file, index, blacklistMode = false) {
this.postsService.deleteFile(file.uid, file.isAudio, blacklistMode).subscribe(result => {
deleteNormalFile(file, blacklistMode = false) {
this.postsService.deleteFile(file.uid, blacklistMode).subscribe(result => {
if (result) {
this.postsService.openSnackBar('Delete success!', 'OK.');
this.files.splice(index, 1);
this.removeFileCard(file);
} else {
this.postsService.openSnackBar('Delete failed!', 'OK.');
}
@@ -248,27 +286,50 @@ export class RecentVideosComponent implements OnInit {
});
}
deleteSubscriptionFile(file, index, blacklistMode = false) {
deleteSubscriptionFile(file, blacklistMode = false) {
if (blacklistMode) {
this.deleteForever(file, index);
this.deleteForever(file);
} else {
this.deleteAndRedownload(file, index);
this.deleteAndRedownload(file);
}
}
deleteAndRedownload(file, index) {
deleteAndRedownload(file) {
const sub = this.postsService.getSubscriptionByID(file.sub_id);
this.postsService.deleteSubscriptionFile(sub, file.id, false, file.uid).subscribe(res => {
this.postsService.openSnackBar(`Successfully deleted file: '${file.id}'`);
this.files.splice(index, 1);
this.removeFileCard(file);
});
}
deleteForever(file, index) {
deleteForever(file) {
const sub = this.postsService.getSubscriptionByID(file.sub_id);
this.postsService.deleteSubscriptionFile(sub, file.id, true, file.uid).subscribe(res => {
this.postsService.openSnackBar(`Successfully deleted file: '${file.id}'`);
this.files.splice(index, 1);
this.removeFileCard(file);
});
}
removeFileCard(file_to_remove) {
const index = this.paged_data.map(e => e.uid).indexOf(file_to_remove.uid);
this.paged_data.splice(index, 1);
this.getAllFiles(true);
}
addFileToPlaylist(info_obj) {
const file = info_obj['file'];
const playlist_id = info_obj['playlist_id'];
const playlist = this.playlists.find(potential_playlist => potential_playlist['id'] === playlist_id);
this.postsService.addFileToPlaylist(playlist_id, file['uid']).subscribe(res => {
if (res['success']) {
this.postsService.openSnackBar(`Successfully added ${file.title} to ${playlist.title}!`);
this.postsService.playlists_changed.next(true);
} else {
this.postsService.openSnackBar(`Failed to add ${file.title} to ${playlist.title}! Unknown error.`);
}
}, err => {
console.error(err);
this.postsService.openSnackBar(`Failed to add ${file.title} to ${playlist.title}! See browser console for error.`);
});
}
@@ -283,9 +344,15 @@ export class RecentVideosComponent implements OnInit {
durationStringToNumber(dur_str) {
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
for (let i = dur_str_parts.length-1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
for (let i = dur_str_parts.length - 1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i]) * (60 ** (dur_str_parts.length - 1 - i));
}
return num_sum;
}
pageChangeEvent(event) {
this.pageSize = event.pageSize;
this.loading_files = Array(this.pageSize).fill(0);
this.getAllFiles();
}
}

11
src/app/components/see-more/see-more.component.html Normal file
View File

@@ -0,0 +1,11 @@
<span class="text" [ngStyle]="{'-webkit-line-clamp': !see_more_active ? line_limit : null}" [innerHTML]="text | linkify"></span>
<span>
<a [routerLink]="" (click)="toggleSeeMore()">
<ng-container *ngIf="!see_more_active" i18n="See more">
See more.
</ng-container>
<ng-container *ngIf="see_more_active" i18n="See less">
See less.
</ng-container>
</a>
</span>

7
src/app/components/see-more/see-more.component.scss Normal file
View File

@@ -0,0 +1,7 @@
.text {
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-box-orient: vertical;
word-wrap: break-word;
}

12
src/app/file-card/file-card.component.spec.ts → src/app/components/see-more/see-more.component.spec.ts
View File

@@ -1,20 +1,20 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { FileCardComponent } from './file-card.component';
import { SeeMoreComponent } from './see-more.component';
describe('FileCardComponent', () => {
let component: FileCardComponent;
let fixture: ComponentFixture<FileCardComponent>;
describe('SeeMoreComponent', () => {
let component: SeeMoreComponent;
let fixture: ComponentFixture<SeeMoreComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [ FileCardComponent ]
declarations: [ SeeMoreComponent ]
})
.compileComponents();
}));
beforeEach(() => {
fixture = TestBed.createComponent(FileCardComponent);
fixture = TestBed.createComponent(SeeMoreComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});

60
src/app/components/see-more/see-more.component.ts Normal file
View File

@@ -0,0 +1,60 @@
import { Component, Input, OnInit, Pipe, PipeTransform } from '@angular/core';
import { DomSanitizer } from '@angular/platform-browser';
@Pipe({ name: 'linkify' })
export class LinkifyPipe implements PipeTransform {
constructor(private _domSanitizer: DomSanitizer) {}
transform(value: any, args?: any): any {
return this._domSanitizer.bypassSecurityTrustHtml(this.stylize(value));
}
// Modify this method according to your custom logic
private stylize(text: string): string {
let stylizedText: string = '';
if (text && text.length > 0) {
for (let line of text.split("\n")) {
for (let t of line.split(" ")) {
if (t.startsWith("http") && t.length>7) {
stylizedText += `<a target="_blank" href="${t}">${t}</a> `;
}
else
stylizedText += t + " ";
}
stylizedText += '<br>';
}
return stylizedText;
}
else return text;
}
}
@Component({
selector: 'app-see-more',
templateUrl: './see-more.component.html',
providers: [LinkifyPipe],
styleUrls: ['./see-more.component.scss']
})
export class SeeMoreComponent implements OnInit {
see_more_active = false;
@Input() text = '';
@Input() line_limit = 2;
constructor() { }
ngOnInit(): void {
}
toggleSeeMore() {
this.see_more_active = !this.see_more_active;
}
parseText() {
return this.text.replace(/(http.*?\s)/, "<a href=\"$1\">$1</a>")
}
}

1
src/app/components/skip-ad-button/skip-ad-button.component.html Normal file
View File

@@ -0,0 +1 @@
<button *ngIf="show_skip_ad_button" (click)="skipAdButtonClicked()" mat-flat-button><ng-container i18n="Skip ad button">Skip ad</ng-container></button>

0
src/app/components/skip-ad-button/skip-ad-button.component.scss Normal file
View File

25
src/app/components/skip-ad-button/skip-ad-button.component.spec.ts Normal file
View File

@@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { SkipAdButtonComponent } from './skip-ad-button.component';
describe('SkipAdButtonComponent', () => {
let component: SkipAdButtonComponent;
let fixture: ComponentFixture<SkipAdButtonComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ SkipAdButtonComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(SkipAdButtonComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

115
src/app/components/skip-ad-button/skip-ad-button.component.ts Normal file
View File

@@ -0,0 +1,115 @@
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
import { PostsService } from 'app/posts.services';
import CryptoJS from 'crypto-js';
@Component({
selector: 'app-skip-ad-button',
templateUrl: './skip-ad-button.component.html',
styleUrls: ['./skip-ad-button.component.scss']
})
export class SkipAdButtonComponent implements OnInit {
@Input() current_video = null;
@Input() playback_timestamp = null;
@Output() setPlaybackTimestamp = new EventEmitter<any>();
sponsor_block_cache = {};
show_skip_ad_button = false;
skip_ad_button_check_interval = null;
constructor(private postsService: PostsService) { }
ngOnInit(): void {
this.skip_ad_button_check_interval = setInterval(() => this.skipAdButtonCheck(), 500);
}
ngOnDestroy(): void {
clearInterval(this.skip_ad_button_check_interval);
}
checkSponsorBlock(video_to_check) {
if (!video_to_check) return;
// check cache, null means it has been checked and confirmed not to exist (limits API calls)
if (this.sponsor_block_cache[video_to_check.url] || this.sponsor_block_cache[video_to_check.url] === null) return;
// sponsor block needs first 4 chars from video ID hash
const video_id = this.getVideoIDFromURL(video_to_check.url);
const id_hash = this.getVideoIDHashFromURL(video_id);
if (!id_hash || id_hash.length < 4) return;
const truncated_id_hash = id_hash.substring(0, 4);
// we couldn't get the data from the cache, let's get it from sponsor block directly
this.postsService.getSponsorBlockDataForVideo(truncated_id_hash).subscribe(res => {
if (res && res['length'] && res['length'] === 0) {
return;
}
const found_data = res['find'](data => data['videoID'] === video_id);
if (found_data) {
this.sponsor_block_cache[video_to_check.url] = found_data;
} else {
this.sponsor_block_cache[video_to_check.url] = null;
}
}, err => {
// likely doesn't exist
this.sponsor_block_cache[video_to_check.url] = null;
});
}
getVideoIDHashFromURL(video_id) {
if (!video_id) return null;
return CryptoJS.SHA256(video_id).toString(CryptoJS.enc.Hex);;
}
getVideoIDFromURL(url) {
const regex_exp = /^.*((youtu.be\/)|(v\/)|(\/u\/\w\/)|(embed\/)|(watch\?))\??v?=?([^#&?]*).*/;
const match = url.match(regex_exp);
return (match && match[7].length==11) ? match[7] : null;
}
skipAdButtonCheck() {
const sponsor_block_data = this.sponsor_block_cache[this.current_video.url];
if (!sponsor_block_data && sponsor_block_data !== null) {
// we haven't yet tried to get the sponsor block data for the video
this.checkSponsorBlock(this.current_video);
} else if (!sponsor_block_data) {
this.show_skip_ad_button = false;
return;
}
if (this.getTimeToSkipTo()) {
this.show_skip_ad_button = true;
} else {
this.show_skip_ad_button = false;
}
}
getTimeToSkipTo() {
const sponsor_block_data = this.sponsor_block_cache[this.current_video.url];
if (!sponsor_block_data) return;
// check if we're in between an ad segment
const found_segment = sponsor_block_data['segments'].find(segment_data => this.playback_timestamp > segment_data.segment[0] && this.playback_timestamp < segment_data.segment[1] - 0.5);
if (found_segment) {
return found_segment['segment'][1];
}
return null;
}
skipAdButtonClicked() {
const time_to_skip_to = this.getTimeToSkipTo();
if (!time_to_skip_to) return;
this.setPlaybackTimestamp.emit(time_to_skip_to);
this.show_skip_ad_button = false;
}
}

Some files were not shown because too many files have changed in this diff Show More