TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into patch-1

Browse Source
This commit is contained in:
Glassed Silver
2022-03-25 10:41:31 +01:00
committed by GitHub
parent 7e9d1d30da 3538132d24
commit ee5d6dfba8
244 changed files with 52277 additions and 7809 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.eslintrc.json Normal file
View File

@@ -0,0 +1,20 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
}

38
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,38 @@
---
name: Bug report
about: Create a report to help us improve
title: "[BUG]"
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Environment**
- YoutubeDL-Material version
- Docker tag: <tag> (optional)
Ideally you'd copy the info as presented on the "About" dialogue
in YoutubeDL-Material.
(for that, click on the three dots on the top right and then
check "installation details". On later versions of YoutubeDL-
Material you will find pretty much all the crucial information
here that we need in most cases!)
**Additional context**
Add any other context about the problem here. For example, a YouTube link.

17
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@@ -0,0 +1,17 @@
---
name: Feature request
about: Suggest an idea for this project
title: "[FEATURE]"
labels: enhancement
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Additional context**
Add any other context or screenshots about the feature request here.

22
.github/workflows/build.yml vendored
View File

@@ -15,15 +15,31 @@ jobs:
- name: checkout code
uses: actions/checkout@v2
- name: setup node
uses: actions/setup-node@v1
uses: actions/setup-node@v2
with:
node-version: '12'
cache: 'npm'
- name: install dependencies
run: |
npm install
cd backend
npm install
sudo npm install -g @angular/cli
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: build
run: ng build --prod
run: npm run build
- name: prepare artifact upload
shell: pwsh
run: |
@@ -35,7 +51,7 @@ jobs:
Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
New-Item -Path ./build/youtubedl-material -Name users
New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
- name: upload build artifact

13
.github/workflows/docker-release.yml vendored
View File

@@ -13,6 +13,19 @@ jobs:
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build

13
.github/workflows/docker.yml vendored
View File

@@ -10,6 +10,19 @@ jobs:
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build

3
.gitignore vendored
View File

@@ -65,4 +65,5 @@ backend/appdata/logs/error.log
backend/appdata/users.json
backend/users/*
backend/appdata/cookies.txt
backend/public
backend/public
src/assets/i18n/*.json

25
.vscode/tasks.json vendored Normal file
View File

@@ -0,0 +1,25 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "npm",
"script": "start",
"problemMatcher": [],
"label": "Dev: start frontend",
"detail": "ng serve"
},
{
"label": "Dev: start backend",
"type": "shell",
"command": "set YTDL_MODE=debug && node app.js",
"options": {
"cwd": "./backend"
},
"presentation": {
"reveal": "always",
"panel": "new"
},
"problemMatcher": []
}
]
}

13
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM alpine:3.12 as frontend
FROM alpine:latest as frontend
RUN apk add --no-cache \
npm
@@ -11,28 +11,33 @@ RUN npm install
COPY [ "angular.json", "tsconfig.json", "/build/" ]
COPY [ "src/", "/build/src/" ]
RUN ng build --prod
RUN npm run build
#--------------#
FROM alpine:3.12
FROM alpine:latest
ENV UID=1000 \
GID=1000 \
USER=youtube
ENV NO_UPDATE_NOTIFIER=true
RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
RUN apk add --no-cache \
ffmpeg \
npm \
python2 \
python3 \
su-exec \
&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
atomicparsley
WORKDIR /app
COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
ENV PM2_HOME=/app/pm2
RUN npm install pm2 -g
RUN npm install && chown -R $UID:$GID ./
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
@@ -40,4 +45,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "node", "app.js" ]
CMD [ "pm2-runtime", "pm2.config.js" ]

1899
Public API v1.yaml
View File

File diff suppressed because it is too large Load Diff

16
README.md
View File

@@ -29,7 +29,7 @@ NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker
Debian/Ubuntu:
```bash
sudo apt-get install nodejs youtube-dl ffmpeg
sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
```
CentOS 7:
@@ -67,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
@@ -77,6 +77,10 @@ Alternatively, you can port forward the port specified in the config (defaults t
## Docker
### Host-specific instructions
If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
### Setup
If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
@@ -106,6 +110,12 @@ To get started, go to the settings menu and enable the public API from the *Extr
Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above.
## iOS Shortcut
If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?)
You can download Shortcut [here.](https://routinehub.co/shortcut/10283/)
## Contributing
If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away.
@@ -124,7 +134,7 @@ Official translators:
* German - UnlimitedCookies
* Chinese - TyRoyal
See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
## License

18
backend/.eslintrc.json Normal file
View File

@@ -0,0 +1,18 @@
{
"env": {
"node": true,
"es2021": true
},
"extends": [
"eslint:recommended"
],
"parser": "esprima",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [],
"rules": {
},
"root": true
}

2101
backend/app.js
View File

File diff suppressed because it is too large Load Diff

18
backend/appdata/default.json
View File

@@ -12,14 +12,16 @@
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
@@ -30,7 +32,9 @@
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",
@@ -54,6 +58,10 @@
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": true,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,
@@ -61,8 +69,8 @@
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"jwt_expiration": 86400,
"jwt_expiration": 86400,
"logger_level": "info"
}
}
}
}

272
backend/authentication/auth.js
View File

@@ -1,12 +1,10 @@
const path = require('path');
const config_api = require('../config');
const consts = require('../consts');
var subscriptions_api = require('../subscriptions')
const fs = require('fs-extra');
var jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
var bcrypt = require('bcryptjs');
const logger = require('../logger');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -14,17 +12,14 @@ var JwtStrategy = require('passport-jwt').Strategy,
ExtractJwt = require('passport-jwt').ExtractJwt;
// other required vars
let logger = null;
let db = null;
let users_db = null;
let db_api = null;
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function(input_db, input_users_db, input_logger) {
setLogger(input_logger)
setDB(input_db, input_users_db);
exports.initialize = function(db_api) {
setDB(db_api);
/*************************
* Authentication module
@@ -34,21 +29,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
SERVER_SECRET = null;
if (users_db.get('jwt_secret').value()) {
SERVER_SECRET = users_db.get('jwt_secret').value();
if (db_api.users_db.get('jwt_secret').value()) {
SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
} else {
SERVER_SECRET = uuid();
users_db.set('jwt_secret', SERVER_SECRET).write();
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
/*opts.issuer = 'example.com';
opts.audience = 'example.com';*/
exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) {
const user = users_db.get('users').find({uid: jwt_payload.user}).value();
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
} else {
@@ -58,13 +51,8 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
}));
}
function setLogger(input_logger) {
logger = input_logger;
}
function setDB(input_db, input_users_db) {
db = input_db;
users_db = input_users_db;
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport');
@@ -80,7 +68,7 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = function(req, res) {
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
@@ -98,20 +86,20 @@ exports.registerUser = function(req, res) {
}
bcrypt.hash(plaintextPassword, saltRounds)
.then(function(hash) {
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (users_db.get('users').find({uid: userid}).value()) {
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (users_db.get('users').find({name: username}).value()) {
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
@@ -144,16 +132,18 @@ exports.registerUser = function(req, res) {
************************************************/
exports.login = async (username, password) => {
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); return false }
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
}
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) {
return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
}
return done(null, await exports.login(username, password));
}
));
@@ -164,17 +154,17 @@ var getLDAPConfiguration = function(req, callback) {
};
exports.passport.use(new LdapStrategy(getLDAPConfiguration,
function(user, done) {
async function(user, done) {
// check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false);
const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value();
let db_user = await db_api.getRecord('users', {uid: user_uid});
if (!db_user) {
// generate DB user
let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
db_user = new_user;
logger.verbose(`Generated new user ${user_uid} using LDAP`);
}
@@ -198,11 +188,11 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.returnAuthResponse = function(req, res) {
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
token: req.token,
permissions: exports.userPermissions(req.user.uid),
permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -215,7 +205,7 @@ exports.returnAuthResponse = function(req, res) {
* It also passes the user object to the next
* middleware through res.locals
**************************************/
exports.ensureAuthenticatedElseError = function(req, res, next) {
exports.ensureAuthenticatedElseError = (req, res, next) => {
var token = getToken(req.query);
if( token ) {
try {
@@ -233,10 +223,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
}
// change password
exports.changeUserPassword = async function(user_uid, new_pass) {
exports.changeUserPassword = async (user_uid, new_pass) => {
try {
const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
return true;
} catch (err) {
return false;
@@ -244,16 +234,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
}
// change user permissions
exports.changeUserPermissions = function(user_uid, permission, new_value) {
exports.changeUserPermissions = async (user_uid, permission, new_value) => {
try {
const user_db_obj = users_db.get('users').find({uid: user_uid});
user_db_obj.get('permissions').pull(permission).write();
user_db_obj.get('permission_overrides').pull(permission).write();
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
if (new_value === 'yes') {
user_db_obj.get('permissions').push(permission).write();
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} else if (new_value === 'no') {
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
}
return true;
} catch (err) {
@@ -263,12 +252,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
}
// change role permissions
exports.changeRolePermissions = function(role, permission, new_value) {
exports.changeRolePermissions = async (role, permission, new_value) => {
try {
const role_db_obj = users_db.get('roles').get(role);
role_db_obj.get('permissions').pull(permission).write();
await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
if (new_value === 'yes') {
role_db_obj.get('permissions').push(permission).write();
await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
}
return true;
} catch (err) {
@@ -277,19 +265,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
}
}
exports.adminExists = function() {
return !!users_db.get('users').find({uid: 'admin'}).value();
exports.adminExists = async function() {
return !!(await db_api.getRecord('users', {uid: 'admin'}));
}
// video stuff
exports.getUserVideos = function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value();
return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files'];
exports.getUserVideos = async function(user_uid, type) {
const files = await db_api.getRecords('files', {user_uid: user_uid});
return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
}
exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -297,58 +285,17 @@ exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
return file;
}
exports.addPlaylist = function(user_uid, new_playlist) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
exports.removePlaylist = async function(user_uid, playlistID) {
await db_api.removeRecord('playlist', {playlistID: playlistID});
return true;
}
exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
return true;
exports.getUserPlaylists = async function(user_uid) {
return await db_api.getRecords('playlists', {user_uid: user_uid});
}
exports.removePlaylist = function(user_uid, playlistID) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write();
return true;
}
exports.getUserPlaylists = function(user_uid, user_files = null) {
const user = users_db.get('users').find({uid: user_uid}).value();
const playlists = JSON.parse(JSON.stringify(user['playlists']));
const categories = db.get('categories').value();
if (categories && user_files) {
categories.forEach(category => {
const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio);
const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio);
if (audio_files && audio_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: audio_files[0].thumbnailURL,
thumbnailPath: audio_files[0].thumbnailPath,
fileNames: audio_files.map(file => file.id),
type: 'audio',
uid: user_uid,
auto: true
});
}
if (video_files && video_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: video_files[0].thumbnailURL,
thumbnailPath: video_files[0].thumbnailPath,
fileNames: video_files.map(file => file.id),
type: 'video',
uid: user_uid,
auto: true
});
}
});
}
return playlists;
}
exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) {
let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value();
exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
let playlist = await db_api.getRecord('playlists', {id: playlistID});
// prevent unauthorized users from accessing the file info
if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -356,109 +303,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false)
return playlist;
}
exports.registerUserFile = function(user_uid, file_object) {
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
path: file_object['path']
}).write();
users_db.get('users').find({uid: user_uid}).get(`files`)
.push(file_object)
.write();
}
exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) {
exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
if (file_obj) {
const type = file_obj.isAudio ? 'audio' : 'video';
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
// close descriptors
if (config_api.descriptors[file_obj.id]) {
try {
for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) {
config_api.descriptors[file_obj.id][i].destroy();
}
} catch(e) {
}
}
const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext);
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
uid: file_uid
}).write();
if (await fs.pathExists(full_path)) {
// remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null;
if (await fs.pathExists(json_path)) {
youtube_id = await fs.readJSON(json_path).id;
await fs.unlink(json_path);
} else if (await fs.pathExists(alternate_json_path)) {
youtube_id = await fs.readJSON(alternate_json_path).id;
await fs.unlink(alternate_json_path);
}
await fs.unlink(full_path);
// do archive stuff
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
} else {
logger.info(`Could not find archive file for ${type} files. Creating...`);
await fs.ensureFile(archive_path);
}
}
}
success = true;
} else {
success = false;
logger.warn(`User file ${file_uid} does not exist!`);
}
is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
success = true;
return success;
}
exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid});
if (user_db_obj.value()) {
const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid});
if (file_db_obj.value()) {
success = true;
file_db_obj.assign({sharingEnabled: enabled}).write();
}
}
exports.userHasPermission = async function(user_uid, permission) {
return success;
}
exports.userHasPermission = function(user_uid, permission) {
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return false;
}
const role_permissions = (users_db.get('roles').value())['permissions'];
const role_permissions = (await db_api.getRecords('roles'))['permissions'];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -481,16 +342,17 @@ exports.userHasPermission = function(user_uid, permission) {
}
}
exports.userPermissions = function(user_uid) {
exports.userPermissions = async function(user_uid) {
let user_permissions = [];
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return null;
}
const role_permissions = users_db.get('roles').get(role).get('permissions').value()
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];

73
backend/categories.js
View File

@@ -1,16 +1,12 @@
const config_api = require('./config');
const utils = require('./utils');
const logger = require('./logger');
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
function initialize(input_db_api) {
setDB(input_db_api);
}
/*
@@ -38,10 +34,9 @@ async function categorize(file_jsons) {
if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
let selected_category = null;
const categories = getCategories();
const categories = await getCategories();
if (!categories) {
logger.warn('Categories could not be found. Initializing categories...');
db.assign({categories: []}).write();
logger.warn('Categories could not be found.');
return null;
}
@@ -63,11 +58,29 @@ async function categorize(file_jsons) {
return selected_category;
}
function getCategories() {
const categories = db.get('categories').value();
async function getCategories() {
const categories = await db_api.getRecords('categories');
return categories ? categories : null;
}
async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = await getCategories();
if (available_categories && files) {
for (let category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
category['thumbnailPath'] = files_that_match[0].thumbnailPath;
category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
category['id'] = category['uid'];
categories_as_playlists.push(category);
}
}
}
return categories_as_playlists;
}
function applyCategoryRules(file_json, rules, category_name) {
let rules_apply = false;
for (let i = 0; i < rules.length; i++) {
@@ -78,10 +91,10 @@ function applyCategoryRules(file_json, rules, category_name) {
switch (rule['comparator']) {
case 'includes':
rule_applies = file_json[rule['property']].includes(rule['value']);
rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
break;
case 'not_includes':
rule_applies = !(file_json[rule['property']].includes(rule['value']));
rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
break;
case 'equals':
rule_applies = file_json[rule['property']] === rule['value'];
@@ -107,23 +120,25 @@ function applyCategoryRules(file_json, rules, category_name) {
return rules_apply;
}
async function addTagToVideo(tag, video, user_uid) {
// TODO: Implement
}
// async function addTagToVideo(tag, video, user_uid) {
// // TODO: Implement
// }
async function removeTagFromVideo(tag, video, user_uid) {
// TODO: Implement
}
// async function removeTagFromVideo(tag, video, user_uid) {
// // TODO: Implement
// }
// adds tag to list of existing tags (used for tag suggestions)
async function addTagToExistingTags(tag) {
const existing_tags = db.get('tags').value();
if (!existing_tags.includes(tag)) {
db.get('tags').push(tag).write();
}
}
// // adds tag to list of existing tags (used for tag suggestions)
// async function addTagToExistingTags(tag) {
// const existing_tags = db.get('tags').value();
// if (!existing_tags.includes(tag)) {
// db.get('tags').push(tag).write();
// }
// }
module.exports = {
initialize: initialize,
categorize: categorize,
getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists
}

30
backend/config.js
View File

@@ -1,3 +1,5 @@
const logger = require('./logger');
const fs = require('fs');
let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
@@ -5,11 +7,7 @@ const debugMode = process.env.YTDL_MODE === 'debug';
let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
var logger = null;
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_logger) {
setLogger(input_logger);
function initialize() {
ensureConfigFileExists();
ensureConfigItemsExist();
}
@@ -97,13 +95,13 @@ function getConfigItem(key) {
}
let path = CONFIG_ITEMS[key]['path'];
const val = Object.byString(config_json, path);
if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
return Object.byString(DEFAULT_CONFIG, path);
}
return Object.byString(config_json, path);
};
}
function setConfigItem(key, value) {
let success = false;
@@ -175,7 +173,7 @@ module.exports = {
globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
}
DEFAULT_CONFIG = {
const DEFAULT_CONFIG = {
"YoutubeDLMaterial": {
"Host": {
"url": "http://example.com",
@@ -189,14 +187,16 @@ DEFAULT_CONFIG = {
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
@@ -207,7 +207,9 @@ DEFAULT_CONFIG = {
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",
@@ -216,7 +218,7 @@ DEFAULT_CONFIG = {
"Subscriptions": {
"allow_subscriptions": true,
"subscriptions_base_path": "subscriptions/",
"subscriptions_check_interval": "300",
"subscriptions_check_interval": "86400",
"redownload_fresh_uploads": false
},
"Users": {
@@ -231,6 +233,10 @@ DEFAULT_CONFIG = {
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": true,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,

49
backend/consts.js
View File

@@ -1,4 +1,4 @@
let CONFIG_ITEMS = {
exports.CONFIG_ITEMS = {
// Host
'ytdl_url': {
'key': 'ytdl_url',
@@ -42,6 +42,14 @@ let CONFIG_ITEMS = {
'key': 'ytdl_include_metadata',
'path': 'YoutubeDLMaterial.Downloader.include_metadata'
},
'ytdl_max_concurrent_downloads': {
'key': 'ytdl_max_concurrent_downloads',
'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
},
'ytdl_download_rate_limit': {
'key': 'ytdl_download_rate_limit',
'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
},
// Extra
'ytdl_title_top': {
@@ -60,9 +68,9 @@ let CONFIG_ITEMS = {
'key': 'ytdl_download_only_mode',
'path': 'YoutubeDLMaterial.Extra.download_only_mode'
},
'ytdl_allow_multi_download_mode': {
'key': 'ytdl_allow_multi_download_mode',
'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
'ytdl_allow_autoplay': {
'key': 'ytdl_allow_autoplay',
'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
},
'ytdl_enable_downloads_manager': {
'key': 'ytdl_enable_downloads_manager',
@@ -102,6 +110,15 @@ let CONFIG_ITEMS = {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
},
'ytdl_use_sponsorblock_api': {
'key': 'ytdl_use_sponsorblock_api',
'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
},
'ytdl_generate_nfo_files': {
'key': 'ytdl_generate_nfo_files',
'path': 'YoutubeDLMaterial.API.generate_NFO_files'
},
// Themes
'ytdl_default_theme': {
@@ -126,10 +143,6 @@ let CONFIG_ITEMS = {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_check_interval': {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_redownload_fresh_uploads': {
'key': 'ytdl_subscriptions_redownload_fresh_uploads',
'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
@@ -153,6 +166,16 @@ let CONFIG_ITEMS = {
'path': 'YoutubeDLMaterial.Users.ldap_config'
},
// Database
'ytdl_use_local_db': {
'key': 'ytdl_use_local_db',
'path': 'YoutubeDLMaterial.Database.use_local_db'
},
'ytdl_mongodb_connection_string': {
'key': 'ytdl_mongodb_connection_string',
'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
},
// Advanced
'ytdl_default_downloader': {
'key': 'ytdl_default_downloader',
@@ -188,7 +211,7 @@ let CONFIG_ITEMS = {
}
};
AVAILABLE_PERMISSIONS = [
exports.AVAILABLE_PERMISSIONS = [
'filemanager',
'settings',
'subscriptions',
@@ -197,8 +220,6 @@ AVAILABLE_PERMISSIONS = [
'downloads_manager'
];
module.exports = {
CONFIG_ITEMS: CONFIG_ITEMS,
AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS,
CURRENT_VERSION: 'v4.2'
}
exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
exports.CURRENT_VERSION = 'v4.2';

967
backend/db.js
View File

File diff suppressed because it is too large Load Diff

632
backend/downloader.js Normal file
View File

@@ -0,0 +1,632 @@
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const glob = require('glob')
const Mutex = require('async-mutex').Mutex;
const youtubedl = require('youtube-dl');
const logger = require('./logger');
const config_api = require('./config');
const twitch_api = require('./twitch');
const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
let db_api = null;
const mutex = new Mutex();
let should_check_downloads = true;
const archivePath = path.join(__dirname, 'appdata', 'archives');
function setDB(input_db_api) { db_api = input_db_api }
exports.initialize = (input_db_api) => {
setDB(input_db_api);
categories_api.initialize(db_api);
if (db_api.database_initialized) {
setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
});
}
}
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
return await mutex.runExclusive(async () => {
const download = {
url: url,
type: type,
title: '',
user_uid: user_uid,
sub_id: sub_id,
sub_name: sub_name,
options: options,
uid: uuid(),
step_index: 0,
paused: false,
running: false,
finished_step: true,
error: null,
percent_complete: null,
finished: false,
timestamp_start: Date.now()
};
await db_api.insertRecordIntoTable('download_queue', download);
should_check_downloads = true;
return download;
});
}
exports.pauseDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
logger.warn(`Download ${download_uid} is already paused!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be paused before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
}
exports.resumeDownload = async (download_uid) => {
return await mutex.runExclusive(async () => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (!download['paused']) {
logger.warn(`Download ${download_uid} is not paused!`);
return false;
}
const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
should_check_downloads = true;
return success;
})
}
exports.restartDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await exports.clearDownload(download_uid);
const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
should_check_downloads = true;
return success;
}
exports.cancelDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['cancelled']) {
logger.warn(`Download ${download_uid} is already cancelled!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be cancelled before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
}
exports.clearDownload = async (download_uid) => {
return await db_api.removeRecord('download_queue', {uid: download_uid});
}
async function handleDownloadError(download_uid, error_message) {
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
}
async function setupDownloads() {
await fixDownloadState();
setInterval(checkDownloads, 1000);
}
async function fixDownloadState() {
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
for (let i = 0; i < running_downloads.length; i++) {
const running_download = running_downloads[i];
const update_obj = {finished_step: true, paused: true, running: false};
if (running_download['step_index'] > 0) {
update_obj['step_index'] = running_download['step_index'] - 1;
}
await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
}
}
async function checkDownloads() {
if (!should_check_downloads) return;
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
await mutex.runExclusive(async () => {
// avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
if (running_downloads.length === 0) {
should_check_downloads = false;
logger.verbose('Disabling checking downloads as none are available.');
}
return;
});
let running_downloads_count = downloads.filter(download => download['running']).length;
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
for (let i = 0; i < waiting_downloads.length; i++) {
const waiting_download = waiting_downloads[i];
const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
if (waiting_download['finished_step'] && !waiting_download['finished']) {
// move to next step
running_downloads_count++;
if (waiting_download['step_index'] === 0) {
collectInfo(waiting_download['uid']);
} else if (waiting_download['step_index'] === 1) {
downloadQueuedFile(waiting_download['uid']);
}
}
}
}
async function collectInfo(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Collecting info for download ${download_uid}`);
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
if (download['user_uid'] && !options.customFileFolderPath) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, download['user_uid'], type);
options.customFileFolderPath = user_path + path.sep;
}
let args = await exports.generateArgs(url, type, options, download['user_uid']);
// get video info prior to download
let info = await getVideoInfoByURL(url, args, download_uid);
if (!info) {
// info failed, error presumably already recorded
return;
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
info = await getVideoInfoByURL(url, args, download_uid);
}
// setup info required to calculate download progress
const expected_file_size = utils.getExpectedFileSize(info);
const files_to_check_for_progress = [];
// store info in download for future use
if (Array.isArray(info)) {
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
} else {
files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
}
const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
finished_step: true,
running: false,
options: options,
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title']
});
}
async function downloadQueuedFile(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Downloading ${download_uid}`);
return new Promise(async resolve => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
const args = download['args'];
const category = download['category'];
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
if (options.customFileFolderPath) {
fileFolderPath = options.customFileFolderPath;
}
fs.ensureDirSync(fileFolderPath);
const start_time = Date.now();
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
// download file
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
const file_objs = [];
let end_time = Date.now();
let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download_uid, err.stderr);
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download_uid, error_message);
logger.warn(error_message);
resolve(false);
return;
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
const ext = type === 'audio' ? '.mp3' : '.mp4';
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj);
}
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download_uid, error_message);
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
resolve();
}
});
});
}
// helper functions
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const globalArgs = config_api.getConfigItem('ytdl_custom_args');
const useCookies = config_api.getConfigItem('ytdl_use_cookies');
const is_audio = type === 'audio';
let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
const customArgs = options.customArgs;
let customOutput = options.customOutput;
const customQualityConfiguration = options.customQualityConfiguration;
// video-specific args
const selectedHeight = options.selectedHeight;
// audio-specific args
const maxBitrate = options.maxBitrate;
const youtubeUsername = options.youtubeUsername;
const youtubePassword = options.youtubePassword;
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
qualityPath = null;
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
qualityPath = ['-f', 'bestvideo+bestaudio']
}
if (customArgs) {
downloadConfig = customArgs.split(',,');
} else {
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
if (customOutput) {
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
if (qualityPath) downloadConfig.push(...qualityPath);
if (is_audio && !options.skip_audio_args) {
downloadConfig.push('-x');
downloadConfig.push('--audio-format', 'mp3');
}
if (youtubeUsername && youtubePassword) {
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
}
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
// if global args has an output, replce the original output with that of global args
const original_output_index = downloadConfig.indexOf('-o');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
}
if (options.additionalArgs && options.additionalArgs !== '') {
downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
}
// filter out incompatible args
downloadConfig = filterArgs(downloadConfig, is_audio);
if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
return downloadConfig;
}
async function getVideoInfoByURL(url, args = [], download_uid = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
new_args.push('--dump-json');
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
if (output) {
let outputs = [];
try {
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
outputs.push(output_json);
}
resolve(outputs.length === 1 ? outputs[0] : outputs);
} catch(e) {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
await handleDownloadError(download_uid, error);
}
resolve(null);
}
} else {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
await handleDownloadError(download_uid, error_message);
}
resolve(null);
}
});
});
}
function filterArgs(args, isAudio) {
const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
const args_to_remove = isAudio ? video_only_args : audio_only_args;
return args.filter(x => !args_to_remove.includes(x));
}
async function checkDownloadPercent(download_uid) {
/*
This is more of an art than a science, we're just selecting files that start with the file name,
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
be divided by the "total expected bytes."
*/
const download = await db_api.getRecord('download_queue', {uid: download_uid});
const files_to_check_for_progress = download['files_to_check_for_progress'];
const resulting_file_size = download['expected_file_size'];
if (!resulting_file_size) return;
let sum_size = 0;
glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
files.forEach(async file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
});
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
});
}
exports.generateNFOFile = (info, output_path) => {
const nfo_obj = {
episodedetails: {
title: info['fulltitle'],
episode: info['playlist_index'] ? info['playlist_index'] : undefined,
premiered: utils.formatDateString(info['upload_date']),
plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
director: info['artist'] ? info['artist'] : info['uploader']
}
};
const doc = create(nfo_obj);
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join(archivePath);
}
}

8
backend/ecosystem.config.js Normal file
View File

@@ -0,0 +1,8 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder",
watch_delay: 5000
}]
}

2
backend/entrypoint.sh
View File

@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
CMD="node app.js"
CMD="pm2-runtime pm2.config.js"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then

23
backend/logger.js Normal file
View File

@@ -0,0 +1,23 @@
const winston = require('winston');
let debugMode = process.env.YTDL_MODE === 'debug';
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
]
});
module.exports = logger;

1550
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

19
backend/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon -q app.js"
"start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
},
"nodemonConfig": {
"ignore": [
@@ -14,7 +15,8 @@
"public/*"
],
"watch": [
"restart.json"
"restart_update.json",
"restart_general.json"
]
},
"repository": {
@@ -30,7 +32,8 @@
"dependencies": {
"archiver": "^3.1.1",
"async": "^3.1.0",
"axios": "^0.21.1",
"async-mutex": "^0.3.1",
"axios": "^0.21.2",
"bcryptjs": "^2.4.0",
"compression": "^1.7.4",
"config": "^3.2.3",
@@ -43,11 +46,13 @@
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^8.4.0",
"moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"node-fetch": "^2.6.7",
"node-id3": "^0.1.14",
"nodemon": "^2.0.2",
"nodemon": "^2.0.7",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",
@@ -56,10 +61,12 @@
"progress": "^2.0.3",
"ps-node": "^0.1.6",
"read-last-lines": "^1.7.2",
"rxjs": "^7.3.0",
"shortid": "^2.2.15",
"unzipper": "^0.10.10",
"uuidv4": "^6.0.6",
"winston": "^3.2.1",
"winston": "^3.3.3",
"xmlbuilder2": "^3.0.2",
"youtube-dl": "^3.0.2"
}
}

9
backend/pm2.config.js Normal file
View File

@@ -0,0 +1,9 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder",
out_file: "/dev/null",
error_file: "/dev/null"
}]
}

380
backend/subscriptions.js
View File

@@ -1,27 +1,21 @@
const FileSync = require('lowdb/adapters/FileSync')
const fs = require('fs-extra');
const path = require('path');
const youtubedl = require('youtube-dl');
var fs = require('fs-extra');
const { uuid } = require('uuidv4');
var path = require('path');
var youtubedl = require('youtube-dl');
const config_api = require('./config');
const twitch_api = require('./twitch');
var utils = require('./utils');
const utils = require('./utils');
const logger = require('./logger');
const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
let db_api = null;
let downloader_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
function initialize(input_db_api, input_downloader_api) {
setDB(input_db_api);
downloader_api = input_downloader_api;
}
async function subscribe(sub, user_uid = null) {
@@ -34,12 +28,7 @@ async function subscribe(sub, user_uid = null) {
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = false;
if (user_uid)
url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
else
url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
if (!sub.name && url_exists) {
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -48,23 +37,16 @@ async function subscribe(sub, user_uid = null) {
return;
}
// add sub to db
let sub_db = null;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
let success = await getSubscriptionInfo(sub, user_uid);
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub);
if (success) {
sub = sub_db.value();
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
};
}
result_obj.success = success;
result_obj.sub = sub;
@@ -73,13 +55,7 @@ async function subscribe(sub, user_uid = null) {
}
async function getSubscriptionInfo(sub, user_uid = null) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
async function getSubscriptionInfo(sub) {
// get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
@@ -91,8 +67,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, function(err, output) {
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id);
}
@@ -122,31 +98,10 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
// if it's now valid, update
if (sub.name) {
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
}
}
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !sub.archive) {
// must create the archive
const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
const archive_path = path.join(archive_dir, 'archive.txt');
// creates archive directory and text file if it doesn't exist
fs.ensureDirSync(archive_dir);
fs.ensureFileSync(archive_path);
// updates subscription
sub.archive = archive_dir;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
}
// TODO: get even more info
resolve(true);
@@ -163,13 +118,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let result_obj = { success: false, error: '' };
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
const sub_files = await db_api.getRecords('files', {sub_id: id});
for (let i = 0; i < sub_files.length; i++) {
const sub_file = sub_files[i];
if (config_api.descriptors[sub_file['uid']]) {
try {
for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
config_api.descriptors[sub_file['uid']][i].destroy();
}
} catch(e) {
continue;
}
}
}
await db_api.removeRecord('subscriptions', {id: id});
await db_api.removeAllRecords('files', {sub_id: id});
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
@@ -191,20 +158,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
// TODO: combine this with deletefile
let basePath = null;
let sub_db = null;
if (user_uid) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
sub_db = db.get('subscriptions').find({id: sub.id});
}
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
await db_api.removeRecord('files', {uid: file_uid});
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
@@ -243,7 +206,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
utils.removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
@@ -255,14 +218,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
}
async function getVideosForSub(sub, user_uid = null) {
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
const latest_sub_obj = sub_db.value();
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -277,42 +233,39 @@ async function getVideosForSub(sub, user_uid = null) {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
fs.ensureDirSync(appendedBasePath);
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
return new Promise(resolve => {
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
fs.appendFileSync(archive_path, output['id']);
}
}
}
// TODO: reimplement
// const outputs = err.stdout.split(/\r\n|\r|\n/);
// for (let i = 0; i < outputs.length; i++) {
// const output = JSON.parse(outputs[i]);
// await handleOutputJSON(sub, output, i === 0, multiUserMode)
// if (err.stderr.includes(output['id']) && archive_path) {
// // we found a video that errored! add it to the archive to prevent future errors
// if (sub.archive) {
// archive_dir = sub.archive;
// archive_path = path.join(archive_dir, 'archive.txt')
// fs.appendFileSync(archive_path, output['id']);
// }
// }
// }
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
@@ -325,21 +278,30 @@ async function getVideosForSub(sub, user_uid = null) {
resolve(true);
return;
}
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
output_jsons.push(output_json);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
}
resolve(files_to_download);
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
@@ -354,6 +316,26 @@ async function getVideosForSub(sub, user_uid = null) {
});
}
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const base_download_options = {
selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
customFileFolderPath: getAppendedBasePath(sub, basePath),
customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
additionalArgs: sub.custom_args
}
return base_download_options;
}
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
// get basePath
let basePath = null;
@@ -366,14 +348,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
let appendedBasePath = getAppendedBasePath(sub, basePath);
let fullOutput = `'${appendedBasePath}/%(title)s.%(ext)s'`;
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
if (desired_path) {
fullOutput = `'${desired_path}.%(ext)s'`;
fullOutput = `"${desired_path}.%(ext)s"`;
} else if (sub.custom_output) {
fullOutput = `'${appendedBasePath}/${sub.custom_output}.%(ext)s'`;
fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
}
let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
@@ -388,7 +372,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...qualityPath)
if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,');
const customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
@@ -430,109 +414,72 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push('--write-thumbnail');
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
return downloadConfig;
}
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
} else {
path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object);
if (sub_db.get('videos').find({path: path_string}).value()) {
// file already exists in DB, return early to avoid reseting the download date
return;
}
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
const file_name = path.basename(output_json['_filename']);
const id = file_name.substring(0, file_name.length-4);
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, id, sub.type, multiUserMode.user, sub);
async function getFilesToDownload(sub, output_jsons) {
const files_to_download = [];
for (let i = 0; i < output_jsons.length; i++) {
const output_json = output_jsons[i];
const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
if (file_missing) {
const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
}
files_to_download.push(output_json);
}
}
return files_to_download;
}
function getSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
else
return db.get('subscriptions').value();
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
function getAllSubscriptions() {
let subscriptions = null;
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (multiUserMode) {
subscriptions = [];
let users = users_db.get('users').value();
for (let i = 0; i < users.length; i++) {
if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']);
}
} else {
subscriptions = getSubscriptions();
}
return subscriptions;
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}
function getSubscription(subID, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return db.get('subscriptions').find({id: subID}).value();
async function getSubscription(subID) {
return await db_api.getRecord('subscriptions', {id: subID});
}
function getSubscriptionByName(subName, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value();
else
return db.get('subscriptions').find({name: subName}).value();
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
function updateSubscription(sub, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(sub).write();
}
async function updateSubscription(sub) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
return true;
}
function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(sub => {
updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
});
}
function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
}
async function updateSubscriptionProperty(sub, assignment_obj) {
// TODO: combine with updateSubscription
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
return true;
}
function subExists(subID, user_uid = null) {
if (user_uid)
return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return !!db.get('subscriptions').find({id: subID}).value();
}
async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
@@ -548,7 +495,7 @@ async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
checkVideoIfBetterExists(video, sub, user_uid)
await checkVideoIfBetterExists(video, sub, user_uid)
}
});
}
@@ -558,7 +505,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
if (err) {
// video is not available anymore for whatever reason
} else if (output) {
@@ -582,37 +529,9 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
// helper functions
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
@@ -623,8 +542,7 @@ module.exports = {
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
removeIDFromArchive : removeIDFromArchive,
setLogger : setLogger,
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
}

1
backend/test/sample.info.json Normal file
View File

File diff suppressed because one or more lines are too long

352
backend/test/tests.js Normal file
View File

@@ -0,0 +1,352 @@
var assert = require('assert');
const low = require('lowdb')
var winston = require('winston');
process.chdir('./backend')
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync('./appdata/db.json');
const db = low(adapter)
const users_adapter = new FileSync('./appdata/users.json');
const users_db = low(users_adapter);
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
let debugMode = process.env.YTDL_MODE === 'debug';
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: 'debug', name: 'console'})
]
});
var auth_api = require('../authentication/auth');
var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db);
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(true);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Transfer to local', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(false);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
});
describe('Export', function() {
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon: http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
});
describe('Multi User', async function() {
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
});
describe('Authentication', function() {
it('login', async function() {
assert(user);
});
});
describe('Video player - normal', function() {
const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
it('Get video', async function() {
const video_obj = db_api.getVideo(video_to_test, 'admin');
assert(video_obj);
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
const playlist_files_to_download = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const playlist_file = await db_api.getVideo(uid, user_to_test);
playlist_files_to_download.push(playlist_file);
}
const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
it('Subscription zip generator', async function() {
const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
assert(sub);
const sub_files_to_download = [];
for (let i = 0; i < sub_videos.length; i++) {
const sub_file = sub_videos[i];
sub_files_to_download.push(sub_file);
}
const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
});
// describe('Video player - subscription', function() {
// const sub_to_test = '';
// const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
// it('Get video', async function() {
// const video_obj = db_api.getVideo(video_to_test, 'admin', );
// assert(video_obj);
// });
// it('Video access - disallowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(!video_obj);
// });
// it('Video access - allowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(video_obj);
// });
// });
});
describe('Downloader', function() {
const downloader_api = require('../downloader');
downloader_api.initialize(db_api);
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const options = {
ui_uid: uuid(),
user: 'admin'
}
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('download_queue');
});
it('Get file info', async function() {
});
it('Download file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Queue file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Pause file', async function() {
});
it('Generate args', async function() {
const args = await downloader_api.generateArgs(url, 'video', options);
console.log(args);
});
it('Generate args - subscription', async function() {
subscriptions_api.initialize(db_api, logger);
const sub = await subscriptions_api.getSubscription(sub_id);
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
console.log(args);
});
it('Generate kodi NFO file', async function() {
const nfo_file_path = './test/sample.nfo';
if (fs.existsSync(nfo_file_path)) {
fs.unlinkSync(nfo_file_path);
}
const sample_json = fs.readJSONSync('./test/sample.info.json');
downloader_api.generateNFOFile(sample_json, nfo_file_path);
assert(fs.existsSync(nfo_file_path), true);
});
});

225
backend/utils.js
View File

@@ -1,6 +1,10 @@
var fs = require('fs-extra')
var path = require('path')
const fs = require('fs-extra')
const path = require('path')
const ffmpeg = require('fluent-ffmpeg');
const config_api = require('./config');
const logger = require('./logger');
const CONSTS = require('./consts')
const archiver = require('archiver');
const is_windows = process.platform === 'win32';
@@ -41,8 +45,7 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
files.push(jsonobj);
continue;
}
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var upload_date = formatDateString(jsonobj.upload_date);
var isaudio = type === 'audio';
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
@@ -52,6 +55,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
return files;
}
async function createContainerZipFile(container_obj, container_file_objs) {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
}
async function createZipFile(zip_file_path, file_paths) {
let output = fs.createWriteStream(zip_file_path);
var archive = archiver('zip', {
gzip: true,
zlib: { level: 9 } // Sets the compression level.
});
archive.on('error', function(err) {
logger.error(err);
throw err;
});
// pipe archive data to the output file
archive.pipe(output);
for (let file_path of file_paths) {
const file_name = path.parse(file_path).base;
archive.file(file_path, {name: file_name})
}
await archive.finalize();
// wait a tiny bit for the zip to reload in fs
await wait(100);
return zip_file_path;
}
function getJSONMp4(name, customPath, openReadPerms = false) {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -84,16 +124,31 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj;
}
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
}
else obj = 0;
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
function getDownloadedThumbnail(file_path) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
let pngPath = file_path_no_extension + '.png';
if (fs.existsSync(jpgPath))
return jpgPath;
@@ -111,10 +166,6 @@ function getExpectedFileSize(input_info_jsons) {
let expected_filesize = 0;
info_jsons.forEach(info_json => {
if (info_json['filesize']) {
expected_filesize += info_json['filesize'];
return;
}
const formats = info_json['format_id'].split('+');
let individual_expected_filesize = 0;
formats.forEach(format_id => {
@@ -130,20 +181,20 @@ function getExpectedFileSize(input_info_jsons) {
return expected_filesize;
}
function fixVideoMetadataPerms(name, type, customPath = null) {
function fixVideoMetadataPerms(file_path, type) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
path.join(customPath, name + ext + '.info.json'),
file_path_no_extension + '.info.json',
file_path_no_extension + ext + '.info.json',
// Thumbnails
path.join(customPath, name + '.webp'),
path.join(customPath, name + '.jpg')
file_path_no_extension + '.webp',
file_path_no_extension + '.jpg'
];
for (const file of files_to_fix) {
@@ -152,18 +203,68 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
}
}
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
function deleteJSONFile(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let json_path = path.join(customPath, name + '.info.json');
let alternate_json_path = path.join(customPath, name + ext + '.info.json');
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
}
function durationStringToNumber(dur_str) {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
for (let i = dur_str_parts.length-1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
}
return num_sum;
}
function getMatchingCategoryFiles(category, files) {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
function getCurrentDownloader() {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
return details_json['downloader'];
}
async function recFindByExt(base,ext,files,result)
{
@@ -193,6 +294,67 @@ function removeFileExtension(filename) {
return filename_parts.join('.');
}
function formatDateString(date_string) {
return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
}
function createEdgeNGrams(str) {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
return str.split(" ").reduce((ngrams, token) => {
if (token.length > minGram) {
for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
ngrams = [...ngrams, token.substr(0, i)]
}
} else {
ngrams = [...ngrams, token]
}
return ngrams
}, []).join(" ")
}
return str
}
// ffmpeg helper functions
async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
let base_ffmpeg_call = ffmpeg(file_path);
if (start) {
base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
}
if (end) {
base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
}
base_ffmpeg_call
.on('end', () => {
logger.verbose(`Cropping for '${file_path}' complete.`);
fs.unlinkSync(file_path);
fs.moveSync(temp_file_path, file_path);
resolve(true);
})
.on('error', (err) => {
logger.error(`Failed to crop ${file_path}.`);
logger.error(err);
resolve(false);
}).save(temp_file_path);
});
}
/**
* setTimeout, but its a promise.
* @param {number} ms
*/
async function wait(ms) {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -215,13 +377,24 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive: removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
File: File
}

23
chart/.helmignore Normal file
View File

@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
chart/Chart.yaml Normal file
View File

@@ -0,0 +1,24 @@
apiVersion: v2
name: youtubedl-material
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "4.2"

22
chart/templates/NOTES.txt Normal file
View File

@@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "youtubedl-material.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "youtubedl-material.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "youtubedl-material.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "youtubedl-material.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}

62
chart/templates/_helpers.tpl Normal file
View File

@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "youtubedl-material.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "youtubedl-material.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "youtubedl-material.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "youtubedl-material.labels" -}}
helm.sh/chart: {{ include "youtubedl-material.chart" . }}
{{ include "youtubedl-material.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "youtubedl-material.selectorLabels" -}}
app.kubernetes.io/name: {{ include "youtubedl-material.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "youtubedl-material.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "youtubedl-material.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

21
chart/templates/appdata-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.appdata.enabled (not .Values.persistence.appdata.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-appdata
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.appdata.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.appdata.size | quote }}
{{- if .Values.persistence.appdata.storageClass }}
{{- if (eq "-" .Values.persistence.appdata.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.appdata.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

21
chart/templates/audio-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.audio.enabled (not .Values.persistence.audio.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-audio
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.audio.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.audio.size | quote }}
{{- if .Values.persistence.audio.storageClass }}
{{- if (eq "-" .Values.persistence.audio.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.audio.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

121
chart/templates/deployment.yaml Normal file
View File

@@ -0,0 +1,121 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "youtubedl-material.fullname" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "youtubedl-material.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "youtubedl-material.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "youtubedl-material.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: 17442
protocol: TCP
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- mountPath: /app/appdata
name: appdata
{{- if .Values.persistence.appdata.subPath }}
subPath: {{ .Values.persistence.appdata.subPath }}
{{- end }}
- mountPath: /app/audio
name: audio
{{- if .Values.persistence.audio.subPath }}
subPath: {{ .Values.persistence.audio.subPath }}
{{- end }}
- mountPath: /app/video
name: video
{{- if .Values.persistence.video.subPath }}
subPath: {{ .Values.persistence.video.subPath }}
{{- end }}
- mountPath: /app/subscriptions
name: subscriptions
{{- if .Values.persistence.subscriptions.subPath }}
subPath: {{ .Values.persistence.subscriptions.subPath }}
{{- end }}
- mountPath: /app/users
name: users
{{- if .Values.persistence.users.subPath }}
subPath: {{ .Values.persistence.users.subPath }}
{{- end }}
volumes:
- name: appdata
{{- if .Values.persistence.appdata.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.appdata.existingClaim }}{{ .Values.persistence.appdata.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-appdata{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: audio
{{- if .Values.persistence.audio.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.audio.existingClaim }}{{ .Values.persistence.audio.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-audio{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: subscriptions
{{- if .Values.persistence.subscriptions.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.subscriptions.existingClaim }}{{ .Values.persistence.subscriptions.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-subscriptions{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: users
{{- if .Values.persistence.users.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.users.existingClaim }}{{ .Values.persistence.users.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-users{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
- name: video
{{- if .Values.persistence.video.enabled}}
persistentVolumeClaim:
claimName: {{ if .Values.persistence.video.existingClaim }}{{ .Values.persistence.video.existingClaim }}{{- else }}{{ template "youtubedl-material.fullname" . }}-video{{- end }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

41
chart/templates/ingress.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "youtubedl-material.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
backend:
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}

15
chart/templates/service.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "youtubedl-material.fullname" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "youtubedl-material.selectorLabels" . | nindent 4 }}

12
chart/templates/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "youtubedl-material.serviceAccountName" . }}
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

21
chart/templates/subscriptions-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.subscriptions.enabled (not .Values.persistence.subscriptions.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-subscriptions
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.subscriptions.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.subscriptions.size | quote }}
{{- if .Values.persistence.subscriptions.storageClass }}
{{- if (eq "-" .Values.persistence.subscriptions.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.subscriptions.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

15
chart/templates/tests/test-connection.yaml Normal file
View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "youtubedl-material.fullname" . }}-test-connection"
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "youtubedl-material.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never

21
chart/templates/users-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.users.enabled (not .Values.persistence.users.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-users
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.users.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.users.size | quote }}
{{- if .Values.persistence.users.storageClass }}
{{- if (eq "-" .Values.persistence.users.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.users.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

21
chart/templates/video-pvc.yaml Normal file
View File

@@ -0,0 +1,21 @@
{{- if and .Values.persistence.video.enabled (not .Values.persistence.video.existingClaim) }}
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: {{ template "youtubedl-material.fullname" . }}-video
labels:
{{- include "youtubedl-material.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.persistence.video.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.video.size | quote }}
{{- if .Values.persistence.video.storageClass }}
{{- if (eq "-" .Values.persistence.video.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.video.storageClass }}"
{{- end }}
{{- end }}
{{- end -}}

153
chart/values.yaml Normal file
View File

@@ -0,0 +1,153 @@
# Default values for youtubedl-material.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: tzahi12345/youtubedl-material
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 17442
ingress:
enabled: false
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: chart-example.local
paths: []
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
persistence:
appdata:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 1Gi
audio:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
video:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
subscriptions:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
users:
enabled: true
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: "-"
##
## If you want to reuse an existing claim, you can pass the name of the PVC using
## the existingClaim variable
# existingClaim: your-claim
# subPath: some-subpath
accessMode: ReadWriteOnce
size: 50Gi
nodeSelector: {}
tolerations: []
affinity: {}

14
docker-compose.yml
View File

@@ -3,6 +3,9 @@ services:
ytdl_material:
environment:
ALLOW_CONFIG_MUTATIONS: 'true'
ytdl_mongodb_connection_string: 'mongodb://ytdl-mongo-db:27017'
ytdl_use_local_db: 'false'
write_ytdl_config: 'true'
restart: always
volumes:
- ./appdata:/app/appdata
@@ -12,4 +15,13 @@ services:
- ./users:/app/users
ports:
- "8998:17442"
image: tzahi12345/youtubedl-material:latest
image: tzahi12345/youtubedl-material:nightly
ytdl-mongo-db:
image: mongo
ports:
- "27017:27017"
logging:
driver: "none"
container_name: mongo-db
volumes:
- ./db/:/data/db

2003
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

18
package.json
View File

@@ -5,12 +5,14 @@
"scripts": {
"ng": "ng",
"start": "ng serve",
"build": "ng build",
"build": "ng build --prod",
"prebuild": "node src/postbuild.mjs",
"heroku-postbuild": "npm install --prefix backend",
"test": "ng test",
"lint": "ng lint",
"e2e": "ng e2e",
"electron": "ng build --base-href ./ && electron ."
"electron": "ng build --base-href ./ && electron .",
"generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
},
"engines": {
"node": "12.3.1",
@@ -33,9 +35,12 @@
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^2.1.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
"filesize": "^6.1.0",
"fingerprintjs2": "^2.1.0",
"fs-extra": "^10.0.0",
"material-icons": "^0.5.4",
"nan": "^2.14.1",
"ng-lazyload-image": "^7.0.1",
"ngx-avatar": "^4.0.0",
@@ -45,6 +50,7 @@
"tslib": "^2.0.0",
"typescript": "~4.0.5",
"web-animations-js": "^2.3.2",
"xliff-to-json": "^1.0.4",
"zone.js": "~0.10.2"
},
"devDependencies": {
@@ -56,16 +62,20 @@
"@types/file-saver": "^2.0.1",
"@types/jasmine": "~3.6.0",
"@types/node": "^12.11.1",
"@typescript-eslint/eslint-plugin": "^4.29.0",
"@typescript-eslint/parser": "^4.29.0",
"codelyzer": "^6.0.0",
"electron": "^8.0.1",
"electron": "^13.6.6",
"eslint": "^7.32.0",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
"karma": "~5.0.0",
"karma": "~6.3.16",
"karma-chrome-launcher": "~3.1.0",
"karma-cli": "~1.0.1",
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"openapi-typescript-codegen": "^0.4.11",
"protractor": "~7.0.0",
"ts-node": "~3.0.4",
"tslint": "~6.1.0"

101
src/api-types/index.ts Normal file
View File

@@ -0,0 +1,101 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
export type { body_19 } from './models/body_19';
export type { body_20 } from './models/body_20';
export type { Category } from './models/Category';
export { CategoryRule } from './models/CategoryRule';
export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermissionsRequest';
export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
export type { ConcurrentStream } from './models/ConcurrentStream';
export type { Config } from './models/Config';
export type { ConfigResponse } from './models/ConfigResponse';
export type { CreateCategoryRequest } from './models/CreateCategoryRequest';
export type { CreateCategoryResponse } from './models/CreateCategoryResponse';
export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
export type { CropFileSettings } from './models/CropFileSettings';
export type { DatabaseFile } from './models/DatabaseFile';
export type { DBInfoResponse } from './models/DBInfoResponse';
export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
export type { DeleteUserRequest } from './models/DeleteUserRequest';
export type { Download } from './models/Download';
export type { DownloadArchiveRequest } from './models/DownloadArchiveRequest';
export type { DownloadFileRequest } from './models/DownloadFileRequest';
export type { DownloadRequest } from './models/DownloadRequest';
export type { DownloadResponse } from './models/DownloadResponse';
export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
export type { File } from './models/File';
export { FileType } from './models/FileType';
export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
export type { GetDownloadRequest } from './models/GetDownloadRequest';
export type { GetDownloadResponse } from './models/GetDownloadResponse';
export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
export type { GetFileFormatsResponse } from './models/GetFileFormatsResponse';
export type { GetFileRequest } from './models/GetFileRequest';
export type { GetFileResponse } from './models/GetFileResponse';
export type { GetFullTwitchChatRequest } from './models/GetFullTwitchChatRequest';
export type { GetFullTwitchChatResponse } from './models/GetFullTwitchChatResponse';
export type { GetLogsRequest } from './models/GetLogsRequest';
export type { GetLogsResponse } from './models/GetLogsResponse';
export type { GetMp3sResponse } from './models/GetMp3sResponse';
export type { GetMp4sResponse } from './models/GetMp4sResponse';
export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
export type { GetRolesResponse } from './models/GetRolesResponse';
export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
export type { GetUsersResponse } from './models/GetUsersResponse';
export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
export type { inline_response_200_15 } from './models/inline_response_200_15';
export type { LoginRequest } from './models/LoginRequest';
export type { LoginResponse } from './models/LoginResponse';
export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse';
export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SharingToggle } from './models/SharingToggle';
export type { SubscribeRequest } from './models/SubscribeRequest';
export type { SubscribeResponse } from './models/SubscribeResponse';
export type { Subscription } from './models/Subscription';
export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
export type { SuccessObject } from './models/SuccessObject';
export type { TableInfo } from './models/TableInfo';
export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
export type { TransferDBRequest } from './models/TransferDBRequest';
export type { TransferDBResponse } from './models/TransferDBResponse';
export type { TwitchChatMessage } from './models/TwitchChatMessage';
export type { UnsubscribeRequest } from './models/UnsubscribeRequest';
export type { UnsubscribeResponse } from './models/UnsubscribeResponse';
export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
export type { UpdaterStatus } from './models/UpdaterStatus';
export type { UpdateServerRequest } from './models/UpdateServerRequest';
export type { UpdateUserRequest } from './models/UpdateUserRequest';
export type { User } from './models/User';
export { UserPermission } from './models/UserPermission';
export type { Version } from './models/Version';
export type { VersionInfoResponse } from './models/VersionInfoResponse';
export { YesNo } from './models/YesNo';

9
src/api-types/models/AddFileToPlaylistRequest.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface AddFileToPlaylistRequest {
file_uid: string;
playlist_id: string;
}

11
src/api-types/models/BaseChangePermissionsRequest.ts Normal file
View File

@@ -0,0 +1,11 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { UserPermission } from './UserPermission';
import { YesNo } from './YesNo';
export interface BaseChangePermissionsRequest {
permission: UserPermission;
new_value: YesNo;
}

15
src/api-types/models/Category.ts Normal file
View File

@@ -0,0 +1,15 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { CategoryRule } from './CategoryRule';
export interface Category {
name?: string;
uid?: string;
rules?: Array<CategoryRule>;
/**
* Overrides file output for downloaded files in category
*/
custom_output?: string;
}

26
src/api-types/models/CategoryRule.ts Normal file
View File

@@ -0,0 +1,26 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CategoryRule {
preceding_operator?: CategoryRule.preceding_operator;
comparator?: CategoryRule.comparator;
}
export namespace CategoryRule {
export enum preceding_operator {
OR = 'or',
AND = 'and',
}
export enum comparator {
INCLUDES = 'includes',
NOT_INCLUDES = 'not_includes',
EQUALS = 'equals',
NOT_EQUALS = 'not_equals',
}
}

9
src/api-types/models/ChangeRolePermissionsRequest.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
role: string;
}

9
src/api-types/models/ChangeUserPermissionsRequest.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
user_uid: string;
}

11
src/api-types/models/CheckConcurrentStreamRequest.ts Normal file
View File

@@ -0,0 +1,11 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CheckConcurrentStreamRequest {
/**
* UID of the concurrent stream
*/
uid: string;
}

9
src/api-types/models/CheckConcurrentStreamResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { ConcurrentStream } from './ConcurrentStream';
export interface CheckConcurrentStreamResponse {
stream: ConcurrentStream;
}

10
src/api-types/models/ConcurrentStream.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface ConcurrentStream {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
}

8
src/api-types/models/Config.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Config {
YoutubeDLMaterial: any;
}

10
src/api-types/models/ConfigResponse.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Config } from './Config';
export interface ConfigResponse {
config_file: Config;
success: boolean;
}

8
src/api-types/models/CreateCategoryRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CreateCategoryRequest {
name: string;
}

10
src/api-types/models/CreateCategoryResponse.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Category } from './Category';
export interface CreateCategoryResponse {
new_category?: Category;
success?: boolean;
}

12
src/api-types/models/CreatePlaylistRequest.ts Normal file
View File

@@ -0,0 +1,12 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface CreatePlaylistRequest {
playlistName: string;
uids: Array<string>;
type: FileType;
thumbnailURL: string;
}

10
src/api-types/models/CreatePlaylistResponse.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Playlist } from './Playlist';
export interface CreatePlaylistResponse {
new_playlist: Playlist;
success: boolean;
}

9
src/api-types/models/CropFileSettings.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CropFileSettings {
cropFileStart: number;
cropFileEnd: number;
}

18
src/api-types/models/DBInfoResponse.ts Normal file
View File

@@ -0,0 +1,18 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { TableInfo } from './TableInfo';
export interface DBInfoResponse {
using_local_db?: boolean;
stats_by_table?: {
files?: TableInfo,
playlists?: TableInfo,
categories?: TableInfo,
subscriptions?: TableInfo,
users?: TableInfo,
roles?: TableInfo,
download_queue?: TableInfo,
};
}

22
src/api-types/models/DatabaseFile.ts Normal file
View File

@@ -0,0 +1,22 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DatabaseFile {
id: string;
title: string;
thumbnailURL: string;
isAudio: boolean;
/**
* In seconds
*/
duration: number;
url: string;
uploader: string;
size: number;
path: string;
upload_date: string;
uid: string;
sharingEnabled?: boolean;
}

8
src/api-types/models/DeleteCategoryRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteCategoryRequest {
category_uid: string;
}

9
src/api-types/models/DeleteMp3Mp4Request.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteMp3Mp4Request {
uid: string;
blacklistMode?: boolean;
}

10
src/api-types/models/DeletePlaylistRequest.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface DeletePlaylistRequest {
playlist_id: string;
type: FileType;
}

15
src/api-types/models/DeleteSubscriptionFileRequest.ts Normal file
View File

@@ -0,0 +1,15 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { SubscriptionRequestData } from './SubscriptionRequestData';
export interface DeleteSubscriptionFileRequest {
file: string;
file_uid?: string;
sub: SubscriptionRequestData;
/**
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
}

8
src/api-types/models/DeleteUserRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteUserRequest {
uid: string;
}

7
src/api-types/models/Dictionary.ts Normal file
View File

@@ -0,0 +1,7 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type Dictionary<T> = {
[key: string]: T;
}

26
src/api-types/models/Download.ts Normal file
View File

@@ -0,0 +1,26 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Download {
uid: string;
ui_uid?: string;
running: boolean;
finished: boolean;
paused: boolean;
finished_step: boolean;
url: string;
type: string;
title: string;
step_index: number;
percent_complete: number;
timestamp_start: number;
/**
* Error text, set if download fails.
*/
error?: string | null;
user_uid?: string;
sub_id?: string;
sub_name?: string;
}

10
src/api-types/models/DownloadArchiveRequest.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DownloadArchiveRequest {
sub: {
archive_dir: string,
};
}

13
src/api-types/models/DownloadFileRequest.ts Normal file
View File

@@ -0,0 +1,13 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DownloadFileRequest {
uid?: string;
uuid?: string;
sub_id?: string;
playlist_id?: string;
url?: string;
type?: string;
}

44
src/api-types/models/DownloadRequest.ts Normal file
View File

@@ -0,0 +1,44 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { CropFileSettings } from './CropFileSettings';
import { FileType } from './FileType';
export interface DownloadRequest {
url: string;
/**
* Video format code. Overrides other quality options.
*/
customQualityConfiguration?: string;
/**
* Custom command-line arguments for youtube-dl. Overrides all other options, except url.
*/
customArgs?: string;
/**
* Additional command-line arguments for youtube-dl. Added to whatever args would normally be used.
*/
additionalArgs?: string;
/**
* Custom output filename template.
*/
customOutput?: string;
/**
* Login with this account ID
*/
youtubeUsername?: string;
/**
* Account password
*/
youtubePassword?: string;
/**
* Height of the video, if known
*/
selectedHeight?: string;
/**
* Specify ffmpeg/avconv audio quality
*/
maxBitrate?: string;
type?: FileType;
cropFileSettings?: CropFileSettings;
}

9
src/api-types/models/DownloadResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface DownloadResponse {
download?: Download;
}

23
src/api-types/models/DownloadTwitchChatByVODIDRequest.ts Normal file
View File

@@ -0,0 +1,23 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
import { Subscription } from './Subscription';
export interface DownloadTwitchChatByVODIDRequest {
/**
* File ID
*/
id: string;
/**
* ID of the VOD
*/
vodId: string;
type: FileType;
/**
* User UID
*/
uuid?: string;
sub?: Subscription;
}

9
src/api-types/models/DownloadTwitchChatByVODIDResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { TwitchChatMessage } from './TwitchChatMessage';
export interface DownloadTwitchChatByVODIDResponse {
chat: Array<TwitchChatMessage>;
}

8
src/api-types/models/DownloadVideosForSubscriptionRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DownloadVideosForSubscriptionRequest {
subID: string;
}

8
src/api-types/models/File.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface File {
id?: string;
}

9
src/api-types/models/FileType.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}

8
src/api-types/models/GenerateArgsResponse.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GenerateArgsResponse {
args?: Array<string>;
}

8
src/api-types/models/GenerateNewApiKeyResponse.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GenerateNewApiKeyResponse {
new_api_key: string;
}

9
src/api-types/models/GetAllCategoriesResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Category } from './Category';
export interface GetAllCategoriesResponse {
categories: Array<Category>;
}

11
src/api-types/models/GetAllDownloadsRequest.ts Normal file
View File

@@ -0,0 +1,11 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetAllDownloadsRequest {
/**
* Filters downloads with the array
*/
uids?: Array<string> | null;
}

9
src/api-types/models/GetAllDownloadsResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface GetAllDownloadsResponse {
downloads?: Array<Download>;
}

14
src/api-types/models/GetAllFilesResponse.ts Normal file
View File

@@ -0,0 +1,14 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
import { Playlist } from './Playlist';
export interface GetAllFilesResponse {
files: Array<DatabaseFile>;
/**
* All video playlists
*/
playlists: Array<Playlist>;
}

9
src/api-types/models/GetAllSubscriptionsResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Subscription } from './Subscription';
export interface GetAllSubscriptionsResponse {
subscriptions: Array<Subscription>;
}

8
src/api-types/models/GetDownloadRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetDownloadRequest {
download_uid: string;
}

9
src/api-types/models/GetDownloadResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface GetDownloadResponse {
download?: Download;
}

8
src/api-types/models/GetFileFormatsRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetFileFormatsRequest {
url?: string;
}

12
src/api-types/models/GetFileFormatsResponse.ts Normal file
View File

@@ -0,0 +1,12 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { File } from './File';
export interface GetFileFormatsResponse {
success: boolean;
result: {
formats?: Array<any>,
};
}

17
src/api-types/models/GetFileRequest.ts Normal file
View File

@@ -0,0 +1,17 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface GetFileRequest {
/**
* Video UID
*/
uid: string;
type?: FileType;
/**
* User UID
*/
uuid?: string;
}

10
src/api-types/models/GetFileResponse.ts Normal file
View File

@@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
export interface GetFileResponse {
success: boolean;
file?: DatabaseFile;
}

22
src/api-types/models/GetFullTwitchChatRequest.ts Normal file
View File

@@ -0,0 +1,22 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
import { Subscription } from './Subscription';
export interface GetFullTwitchChatRequest {
/**
* File ID
*/
id: string;
type: FileType;
/**
* User UID
*/
uuid?: string;
/**
* Subscription
*/
sub?: Subscription;
}

9
src/api-types/models/GetFullTwitchChatResponse.ts Normal file
View File

@@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetFullTwitchChatResponse {
success: boolean;
error?: string;
}

8
src/api-types/models/GetLogsRequest.ts Normal file
View File

@@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetLogsRequest {
lines?: number;
}

12
src/api-types/models/GetLogsResponse.ts Normal file
View File

@@ -0,0 +1,12 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetLogsResponse {
/**
* Number of lines to retrieve from the bottom
*/
logs?: string;
success?: boolean;
}

Some files were not shown because too many files have changed in this diff Show More