Compare commits

..

3 Commits

Author SHA1 Message Date
Isaac Abadi
c789ba9553 Server autocloses on crash
Thumbnails are now retrieved using file UID
2021-07-31 15:51:16 -06:00
Isaac Abadi
b8e1117ff6 Removed all __dirname references in backend to allow for electron to boot 2021-07-28 21:14:32 -06:00
Isaac Abadi
b64a001ae1 Electron almost boots, but errors presumably due to a filesystem issue (missing folder?) 2021-07-28 19:17:08 -06:00
244 changed files with 16692 additions and 39249 deletions

View File

@@ -1,7 +0,0 @@
.git
db
appdata
audio
video
subscriptions
users

View File

@@ -1,20 +0,0 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
}

View File

@@ -27,12 +27,5 @@ If applicable, add screenshots to help explain your problem.
- YoutubeDL-Material version
- Docker tag: <tag> (optional)
Ideally you'd copy the info as presented on the "About" dialogue
in YoutubeDL-Material.
(for that, click on the three dots on the top right and then
check "installation details". On later versions of YoutubeDL-
Material you will find pretty much all the crucial information
here that we need in most cases!)
**Additional context**
Add any other context about the problem here. For example, a YouTube link.
Add any other context about the problem here.

View File

@@ -25,21 +25,8 @@ jobs:
cd backend
npm install
sudo npm install -g @angular/cli
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: build
run: npm run build
run: ng build --prod
- name: prepare artifact upload
shell: pwsh
run: |
@@ -51,7 +38,7 @@ jobs:
Copy-Item -Path ./backend/public -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/subscriptions -Recurse -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/video -Recurse -Destination ./build/youtubedl-material
New-Item -Path ./build/youtubedl-material -Name users -ItemType Directory
New-Item -Path ./build/youtubedl-material -Name users
Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
- name: upload build artifact

View File

@@ -13,19 +13,6 @@ jobs:
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build

View File

@@ -10,19 +10,6 @@ jobs:
steps:
- name: checkout code
uses: actions/checkout@v2
- name: Set hash
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build

12
.gitignore vendored
View File

@@ -25,7 +25,6 @@
!.vscode/extensions.json
# misc
/.angular/cache
/.sass-cache
/connect.lock
/coverage
@@ -66,13 +65,4 @@ backend/appdata/logs/error.log
backend/appdata/users.json
backend/users/*
backend/appdata/cookies.txt
backend/public
src/assets/i18n/*.json
# User Files
db/
appdata/
audio/
video/
subscriptions/
users/
backend/public

View File

@@ -1,9 +1,4 @@
FROM alpine:latest AS ffmpeg
COPY docker-build.sh .
RUN sh ./docker-build.sh
FROM alpine:latest as frontend
FROM alpine:3.12 as frontend
RUN apk add --no-cache \
npm
@@ -16,21 +11,23 @@ RUN npm install
COPY [ "angular.json", "tsconfig.json", "/build/" ]
COPY [ "src/", "/build/src/" ]
RUN npm run build
RUN ng build --prod
#--------------#
FROM alpine:latest
FROM alpine:3.12
ENV UID=1000 \
GID=1000 \
USER=youtube
ENV NO_UPDATE_NOTIFIER=true
ENV FOREVER_ROOT=/app/.forever
RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
RUN apk add --no-cache \
ffmpeg \
npm \
python2 \
python3 \
@@ -39,11 +36,8 @@ RUN apk add --no-cache \
atomicparsley
WORKDIR /app
COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe
COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
ENV PM2_HOME=/app/pm2
RUN npm install pm2 -g
RUN npm install forever -g
RUN npm install && chown -R $UID:$GID ./
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
@@ -51,4 +45,4 @@ COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "pm2-runtime", "pm2.config.js" ]
CMD [ "forever", "app.js" ]

File diff suppressed because it is too large Load Diff

View File

@@ -10,18 +10,6 @@ YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.gi
Now with [Docker](#Docker) support!
<hr>
### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
<hr>
## Getting Started
Check out the prerequisites, and go to the installation section. Easy as pie!
@@ -79,7 +67,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `ng build --prod`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.
@@ -89,10 +77,6 @@ Alternatively, you can port forward the port specified in the config (defaults t
## Docker
### Host-specific instructions
If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
### Setup
If you are looking to setup YoutubeDL-Material with Docker, this section is for you. And you're in luck! Docker setup is quite simple.
@@ -122,12 +106,6 @@ To get started, go to the settings menu and enable the public API from the *Extr
Once you have enabled the API and have the key, you can start sending requests by adding the query param `apiKey=API_KEY`. Replace `API_KEY` with your actual API key, and you should be good to go! Nearly all of the backend should be at your disposal. View available endpoints in the link above.
## iOS Shortcut
If you are using iOS, try YoutubeDL-Material more conveniently with a Shortcut. With this Shorcut, you can easily start downloading YouTube video with just two taps! (Or maybe three?)
You can download Shortcut [here.](https://routinehub.co/shortcut/10283/)
## Contributing
If you're interested in contributing, first: awesome! Second, please refer to the guidelines/setup information located in the [Contributing](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Contributing) wiki page, it's a helpful way to get you on your feet and coding away.
@@ -146,16 +124,12 @@ Official translators:
* German - UnlimitedCookies
* Chinese - TyRoyal
See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
## License
This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details
## Legal Disclaimer
This project is in no way affiliated with Google LLC, Alphabet Inc. or YouTube (or their subsidiaries) nor endorsed by them.
## Acknowledgments
* youtube-dl

View File

@@ -17,6 +17,7 @@
"build": {
"builder": "@angular-devkit/build-angular:browser",
"options": {
"aot": true,
"outputPath": "backend/public",
"index": "src/index.html",
"main": "src/main.ts",
@@ -32,17 +33,7 @@
"styles": [
"src/styles.scss"
],
"scripts": [],
"vendorChunk": true,
"extractLicenses": false,
"buildOptimizer": false,
"sourceMap": true,
"optimization": false,
"namedChunks": true,
"allowedCommonJsDependencies": [
"rxjs",
"crypto-js"
]
"scripts": []
},
"configurations": {
"production": {
@@ -55,6 +46,7 @@
"optimization": true,
"outputHashing": "all",
"namedChunks": false,
"aot": true,
"extractLicenses": true,
"vendorChunk": false,
"buildOptimizer": true,
@@ -68,8 +60,7 @@
"es": {
"localize": ["es"]
}
},
"defaultConfiguration": ""
}
},
"serve": {
"builder": "@angular-devkit/build-angular:dev-server",
@@ -161,6 +152,16 @@
"src/backend"
]
}
},
"lint": {
"builder": "@angular-devkit/build-angular:tslint",
"options": {
"tsConfig": [
"src/tsconfig.app.json",
"src/tsconfig.spec.json"
],
"exclude": []
}
}
}
},
@@ -175,6 +176,15 @@
"protractorConfig": "./protractor.conf.js",
"devServerTarget": "youtube-dl-material:serve"
}
},
"lint": {
"builder": "@angular-devkit/build-angular:tslint",
"options": {
"tsConfig": [
"e2e/tsconfig.e2e.json"
],
"exclude": []
}
}
}
}

View File

@@ -1,18 +0,0 @@
{
"env": {
"node": true,
"es2021": true
},
"extends": [
"eslint:recommended"
],
"parser": "esprima",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [],
"rules": {
},
"root": true
}

File diff suppressed because it is too large Load Diff

View File

@@ -12,16 +12,14 @@
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_autoplay": true,
"allow_multi_download_mode": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
@@ -32,9 +30,7 @@
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
"twitch_auto_download_chat": false
},
"Themes": {
"default_theme": "default",
@@ -59,7 +55,7 @@
}
},
"Database": {
"use_local_db": true,
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
@@ -69,8 +65,8 @@
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"jwt_expiration": 86400,
"jwt_expiration": 86400,
"logger_level": "info"
}
}
}
}

View File

@@ -1,8 +1,7 @@
const path = require('path');
const config_api = require('../config');
const consts = require('../consts');
const logger = require('../logger');
const db_api = require('../db');
const fs = require('fs-extra');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
@@ -13,12 +12,17 @@ var JwtStrategy = require('passport-jwt').Strategy,
ExtractJwt = require('passport-jwt').ExtractJwt;
// other required vars
let logger = null;
let db_api = null;
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function() {
exports.initialize = function(db_api, input_logger) {
setLogger(input_logger)
setDB(db_api);
/*************************
* Authentication module
************************/
@@ -49,6 +53,14 @@ exports.initialize = function() {
}));
}
function setLogger(input_logger) {
logger = input_logger;
}
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport');
exports.passport.serializeUser(function(user, done) {
@@ -128,7 +140,7 @@ exports.registerUser = async function(req, res) {
exports.login = async (username, password) => {
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); return false }
if (!user) { logger.error(`User ${username} not found`); false }
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
}
@@ -279,12 +291,17 @@ exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false
return file;
}
exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
return true;
}
exports.removePlaylist = async function(user_uid, playlistID) {
await db_api.removeRecord('playlist', {playlistID: playlistID});
return true;
}
exports.getUserPlaylists = async function(user_uid) {
exports.getUserPlaylists = async function(user_uid, user_files = null) {
return await db_api.getRecords('playlists', {user_uid: user_uid});
}

View File

@@ -1,6 +1,19 @@
const config_api = require('./config');
const utils = require('./utils');
const logger = require('./logger');
const db_api = require('./db');
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
}
/*
Categories:
@@ -59,7 +72,7 @@ async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = await getCategories();
if (available_categories && files) {
for (let category of available_categories) {
for (category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
@@ -112,23 +125,24 @@ function applyCategoryRules(file_json, rules, category_name) {
return rules_apply;
}
// async function addTagToVideo(tag, video, user_uid) {
// // TODO: Implement
// }
async function addTagToVideo(tag, video, user_uid) {
// TODO: Implement
}
// async function removeTagFromVideo(tag, video, user_uid) {
// // TODO: Implement
// }
async function removeTagFromVideo(tag, video, user_uid) {
// TODO: Implement
}
// // adds tag to list of existing tags (used for tag suggestions)
// async function addTagToExistingTags(tag) {
// const existing_tags = db.get('tags').value();
// if (!existing_tags.includes(tag)) {
// db.get('tags').push(tag).write();
// }
// }
// adds tag to list of existing tags (used for tag suggestions)
async function addTagToExistingTags(tag) {
const existing_tags = db.get('tags').value();
if (!existing_tags.includes(tag)) {
db.get('tags').push(tag).write();
}
}
module.exports = {
initialize: initialize,
categorize: categorize,
getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists

View File

@@ -1,5 +1,3 @@
const logger = require('./logger');
const fs = require('fs');
let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
@@ -7,7 +5,11 @@ const debugMode = process.env.YTDL_MODE === 'debug';
let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
function initialize() {
var logger = null;
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_logger) {
setLogger(input_logger);
ensureConfigFileExists();
ensureConfigItemsExist();
}
@@ -95,13 +97,13 @@ function getConfigItem(key) {
}
let path = CONFIG_ITEMS[key]['path'];
const val = Object.byString(config_json, path);
if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
return Object.byString(DEFAULT_CONFIG, path);
}
return Object.byString(config_json, path);
}
};
function setConfigItem(key, value) {
let success = false;
@@ -173,7 +175,7 @@ module.exports = {
globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
}
const DEFAULT_CONFIG = {
DEFAULT_CONFIG = {
"YoutubeDLMaterial": {
"Host": {
"url": "http://example.com",
@@ -187,16 +189,14 @@ const DEFAULT_CONFIG = {
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true,
"max_concurrent_downloads": 5,
"download_rate_limit": ""
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_autoplay": true,
"allow_multi_download_mode": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
@@ -207,9 +207,7 @@ const DEFAULT_CONFIG = {
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
"twitch_auto_download_chat": false
},
"Themes": {
"default_theme": "default",
@@ -218,7 +216,7 @@ const DEFAULT_CONFIG = {
"Subscriptions": {
"allow_subscriptions": true,
"subscriptions_base_path": "subscriptions/",
"subscriptions_check_interval": "86400",
"subscriptions_check_interval": "300",
"redownload_fresh_uploads": false
},
"Users": {
@@ -234,7 +232,7 @@ const DEFAULT_CONFIG = {
}
},
"Database": {
"use_local_db": true,
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {

View File

@@ -1,4 +1,4 @@
exports.CONFIG_ITEMS = {
let CONFIG_ITEMS = {
// Host
'ytdl_url': {
'key': 'ytdl_url',
@@ -42,14 +42,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_include_metadata',
'path': 'YoutubeDLMaterial.Downloader.include_metadata'
},
'ytdl_max_concurrent_downloads': {
'key': 'ytdl_max_concurrent_downloads',
'path': 'YoutubeDLMaterial.Downloader.max_concurrent_downloads'
},
'ytdl_download_rate_limit': {
'key': 'ytdl_download_rate_limit',
'path': 'YoutubeDLMaterial.Downloader.download_rate_limit'
},
// Extra
'ytdl_title_top': {
@@ -68,9 +60,9 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_download_only_mode',
'path': 'YoutubeDLMaterial.Extra.download_only_mode'
},
'ytdl_allow_autoplay': {
'key': 'ytdl_allow_autoplay',
'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
'ytdl_allow_multi_download_mode': {
'key': 'ytdl_allow_multi_download_mode',
'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
},
'ytdl_enable_downloads_manager': {
'key': 'ytdl_enable_downloads_manager',
@@ -110,15 +102,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
},
'ytdl_use_sponsorblock_api': {
'key': 'ytdl_use_sponsorblock_api',
'path': 'YoutubeDLMaterial.API.use_sponsorblock_API'
},
'ytdl_generate_nfo_files': {
'key': 'ytdl_generate_nfo_files',
'path': 'YoutubeDLMaterial.API.generate_NFO_files'
},
// Themes
'ytdl_default_theme': {
@@ -143,6 +126,10 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_check_interval': {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_redownload_fresh_uploads': {
'key': 'ytdl_subscriptions_redownload_fresh_uploads',
'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'
@@ -211,7 +198,7 @@ exports.CONFIG_ITEMS = {
}
};
exports.AVAILABLE_PERMISSIONS = [
AVAILABLE_PERMISSIONS = [
'filemanager',
'settings',
'subscriptions',
@@ -220,6 +207,8 @@ exports.AVAILABLE_PERMISSIONS = [
'downloads_manager'
];
exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
exports.CURRENT_VERSION = 'v4.2';
module.exports = {
CONFIG_ITEMS: CONFIG_ITEMS,
AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS,
CURRENT_VERSION: 'v4.2'
}

View File

@@ -1,31 +1,24 @@
var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const config_api = require('./config');
var utils = require('./utils')
const logger = require('./logger');
const { uuid } = require('uuidv4');
const config_api = require('./config');
const { MongoClient } = require("mongodb");
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync');
const { BehaviorSubject } = require('rxjs');
const local_adapter = new FileSync('./appdata/local_db.json');
const local_db = low(local_adapter);
let database = null;
exports.database_initialized = false;
exports.database_initialized_bs = new BehaviorSubject(false);
var logger = null;
var db = null;
var users_db = null;
var database = null;
const tables = {
files: {
name: 'files',
primary_key: 'uid',
text_search: {
title: 'text',
uploader: 'text',
uid: 'text'
}
primary_key: 'uid'
},
playlists: {
name: 'playlists',
@@ -50,14 +43,6 @@ const tables = {
name: 'roles',
primary_key: 'key'
},
download_queue: {
name: 'download_queue',
primary_key: 'uid'
},
tasks: {
name: 'tasks',
primary_key: 'key'
},
test: {
name: 'test'
}
@@ -77,15 +62,20 @@ function setDB(input_db, input_users_db) {
exports.users_db = input_users_db
}
exports.initialize = (input_db, input_users_db) => {
function setLogger(input_logger) {
logger = input_logger;
}
exports.initialize = (input_db, input_users_db, input_logger) => {
setDB(input_db, input_users_db);
setLogger(input_logger);
// must be done here to prevent getConfigItem from being called before init
using_local_db = config_api.getConfigItem('ytdl_use_local_db');
}
exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
if (using_local_db && !custom_connection_string) return;
if (using_local_db) return;
const success = await exports._connectToDB(custom_connection_string);
if (success) return true;
@@ -141,13 +131,8 @@ exports._connectToDB = async (custom_connection_string = null) => {
tables_list.forEach(async table => {
const primary_key = tables[table]['primary_key'];
if (primary_key) {
await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
}
const text_search = tables[table]['text_search'];
if (text_search) {
await database.collection(table).createIndex(text_search);
}
if (!primary_key) return;
await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
});
return true;
} catch(err) {
@@ -159,17 +144,51 @@ exports._connectToDB = async (custom_connection_string = null) => {
}
}
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject(file_path, type);
exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => {
let db_path = null;
const file_id = utils.removeFileExtension(file_path);
if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
return false;
}
utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
}
return file_obj;
}
exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject2(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms(file_path, type);
utils.fixVideoMetadataPerms2(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
@@ -186,7 +205,7 @@ exports.registerFileDB = async (file_path, type, user_uid = null, category = nul
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_path, type)
utils.deleteJSONFile2(file_path, type)
}
return file_obj;
@@ -204,12 +223,38 @@ async function registerFileDBManual(file_object) {
return file_object;
}
function generateFileObject(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
function generateFileObject(id, type, customPath = null, sub = null) {
if (!customPath && sub) {
customPath = getAppendedBasePathSub(sub, config_api.getConfigItem('ytdl_subscriptions_base_path'));
}
var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
if (!jsonobj) {
return null;
} else if (!jsonobj['_filename']) {
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
// console.
var stats = fs.statSync(path.join(file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function generateFileObject2(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
@@ -221,7 +266,8 @@ function generateFileObject(file_path, type) {
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = utils.formatDateString(jsonobj.upload_date);
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
var size = stats.size;
@@ -304,7 +350,6 @@ exports.getFileDirectoriesAndDBs = async () => {
}
exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
@@ -317,21 +362,33 @@ exports.importUnregisteredFiles = async () => {
const file = files[j];
// check if file exists in db, if not add it
const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id}))
if (!file_is_registered) {
// add additional info
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
logger.verbose(`Added discovered file to the database: ${file.id}`);
}
}
}
return imported_files;
}
exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
const preimported_file_paths = [];
const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
for (let i = 0; i < files.length; i++) {
const file = files[i];
// check if file exists in db, if not add it
const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
if (!file_is_registered) {
// add additional info
await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
preimported_file_paths.push(file['path']);
logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
}
}
return preimported_file_paths;
}
exports.addMetadataPropertyToDB = async (property_key) => {
@@ -462,8 +519,8 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
jsonPath = path.join(jsonPath);
altJSONPath = path.join(altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
@@ -563,22 +620,7 @@ exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
return true;
}
if (replaceFilter) {
const output = await database.collection(table).bulkWrite([
{
deleteMany: {
filter: replaceFilter
}
},
{
insertOne: {
document: doc
}
}
]);
logger.debug(`Inserted doc into ${table} with filter: ${JSON.stringify(replaceFilter)}`);
return !!(output['result']['ok']);
}
if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
const output = await database.collection(table).insertOne(doc);
logger.debug(`Inserted doc into ${table}`);
@@ -635,28 +677,13 @@ exports.getRecord = async (table, filter_obj) => {
return await database.collection(table).findOne(filter_obj);
}
exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
exports.getRecords = async (table, filter_obj = null) => {
// local db override
if (using_local_db) {
let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
if (sort) {
cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
}
if (range) {
cursor = cursor.slice(range[0], range[1]);
}
return !return_count ? cursor : cursor.length;
return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
}
const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
if (sort) {
cursor.sort({[sort['by']]: sort['order']});
}
if (range) {
cursor.skip(range[0]).limit(range[1] - range[0]);
}
return !return_count ? await cursor.toArray() : await cursor.count();
return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray();
}
// Update
@@ -754,86 +781,26 @@ exports.removeRecord = async (table, filter_obj) => {
return !!(output['result']['ok']);
}
// exports.removeRecordsByUIDBulk = async (table, uids) => {
// // local db override
// if (using_local_db) {
// applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
// return true;
// }
// const table_collection = database.collection(table);
// let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
// const item_ids_to_remove =
// for (let i = 0; i < item_ids_to_update.length; i++) {
// const item_id_to_update = item_ids_to_update[i];
// bulk.find({[key_label]: item_id_to_update }).updateOne({
// "$set": update_obj[item_id_to_update]
// });
// }
// const output = await bulk.execute();
// return !!(output['result']['ok']);
// }
exports.findDuplicatesByKey = async (table, key) => {
let duplicates = [];
if (using_local_db) {
// this can probably be optimized
const all_records = await exports.getRecords(table);
const existing_records = {};
for (let i = 0; i < all_records.length; i++) {
const record = all_records[i];
const value = record[key];
if (existing_records[value]) {
duplicates.push(record);
}
existing_records[value] = true;
}
return duplicates;
}
const duplicated_values = await database.collection(table).aggregate([
{"$group" : { "_id": `$${key}`, "count": { "$sum": 1 } } },
{"$match": {"_id" :{ "$ne" : null } , "count" : {"$gt": 1} } },
{"$project": {[key] : "$_id", "_id" : 0} }
]).toArray();
for (let i = 0; i < duplicated_values.length; i++) {
const duplicated_value = duplicated_values[i];
const duplicated_records = await exports.getRecords(table, duplicated_value, false);
if (duplicated_records.length > 1) {
duplicates = duplicates.concat(duplicated_records.slice(1, duplicated_records.length));
}
}
return duplicates;
}
exports.removeAllRecords = async (table = null, filter_obj = null) => {
exports.removeAllRecords = async (table = null) => {
// local db override
const tables_to_remove = table ? [table] : tables_list;
logger.debug(`Removing all records from: ${tables_to_remove} with filter: ${JSON.stringify(filter_obj)}`)
if (using_local_db) {
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
else local_db.assign({[table_to_remove]: []}).write();
logger.debug(`Successfully removed records from ${table_to_remove}`);
local_db.assign({[table_to_remove]: []}).write();
logger.debug(`Removed all records from ${table_to_remove}`);
}
return true;
}
let success = true;
logger.debug(`Removing all records from: ${tables_to_remove}`)
for (let i = 0; i < tables_to_remove.length; i++) {
const table_to_remove = tables_to_remove[i];
const output = await database.collection(table_to_remove).deleteMany(filter_obj ? filter_obj : {});
logger.debug(`Successfully removed records from ${table_to_remove}`);
const output = await database.collection(table_to_remove).deleteMany({});
logger.debug(`Removed all records from ${table_to_remove}`);
success &= !!(output['result']['ok']);
}
return success;
@@ -987,52 +954,6 @@ const createDownloadsRecords = (downloads) => {
return new_downloads;
}
exports.backupDB = async () => {
const backup_dir = path.join('appdata', 'db_backup');
fs.ensureDirSync(backup_dir);
const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
const path_to_backups = path.join(backup_dir, backup_file_name);
logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
table_to_records[table] = await exports.getRecords(table);
}
fs.writeJsonSync(path_to_backups, table_to_records);
return backup_file_name;
}
exports.restoreDB = async (file_name) => {
const path_to_backup = path.join('appdata', 'db_backup', file_name);
logger.debug('Reading database backup file.');
const table_to_records = fs.readJSONSync(path_to_backup);
if (!table_to_records) {
logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`);
return false;
}
logger.debug('Clearing database.');
await exports.removeAllRecords();
logger.debug('Database cleared! Beginning restore.');
let success = true;
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
}
logger.debug('Restore finished!');
return success;
}
exports.transferDB = async (local_to_remote) => {
const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) {
@@ -1042,8 +963,9 @@ exports.transferDB = async (local_to_remote) => {
using_local_db = !local_to_remote;
if (local_to_remote) {
logger.debug('Backup up DB...');
await exports.backupDB();
// backup local DB
logger.debug('Backup up Local DB...');
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
const db_connected = await exports.connectToDB(5, true);
if (!db_connected) {
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');
@@ -1066,8 +988,6 @@ exports.transferDB = async (local_to_remote) => {
config_api.setConfigItem('ytdl_use_local_db', using_local_db);
logger.debug('Transfer finished!');
return success;
}
@@ -1087,28 +1007,10 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
if (filter_prop_value === undefined || filter_prop_value === null) {
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
} else {
if (typeof filter_prop_value === 'object') {
if (filter_prop_value['$regex']) {
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
}
} else {
filtered &= record[filter_prop] === filter_prop_value;
}
filtered &= record[filter_prop] === filter_prop_value;
}
}
return filtered;
});
return return_val;
}
// archive helper functions
async function writeToBlacklist(type, line) {
const archivePath = path.join(__dirname, 'appdata', 'archives');
let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
// adds newline to the beginning of the line
line.replace('\n', '');
line.replace('\r', '');
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
}

View File

@@ -1,625 +0,0 @@
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const glob = require('glob')
const Mutex = require('async-mutex').Mutex;
const youtubedl = require('youtube-dl');
const logger = require('./logger');
const config_api = require('./config');
const twitch_api = require('./twitch');
const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const mutex = new Mutex();
let should_check_downloads = true;
const archivePath = path.join(__dirname, 'appdata', 'archives');
if (db_api.database_initialized) {
setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
});
}
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
return await mutex.runExclusive(async () => {
const download = {
url: url,
type: type,
title: '',
user_uid: user_uid,
sub_id: sub_id,
sub_name: sub_name,
options: options,
uid: uuid(),
step_index: 0,
paused: false,
running: false,
finished_step: true,
error: null,
percent_complete: null,
finished: false,
timestamp_start: Date.now()
};
await db_api.insertRecordIntoTable('download_queue', download);
should_check_downloads = true;
return download;
});
}
exports.pauseDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
logger.warn(`Download ${download_uid} is already paused!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be paused before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
}
exports.resumeDownload = async (download_uid) => {
return await mutex.runExclusive(async () => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (!download['paused']) {
logger.warn(`Download ${download_uid} is not paused!`);
return false;
}
const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
should_check_downloads = true;
return success;
})
}
exports.restartDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await exports.clearDownload(download_uid);
const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
should_check_downloads = true;
return success;
}
exports.cancelDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['cancelled']) {
logger.warn(`Download ${download_uid} is already cancelled!`);
return false;
} else if (download['finished']) {
logger.info(`Download ${download_uid} could not be cancelled before completing.`);
return false;
}
return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
}
exports.clearDownload = async (download_uid) => {
return await db_api.removeRecord('download_queue', {uid: download_uid});
}
async function handleDownloadError(download_uid, error_message) {
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
}
async function setupDownloads() {
await fixDownloadState();
setInterval(checkDownloads, 1000);
}
async function fixDownloadState() {
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
for (let i = 0; i < running_downloads.length; i++) {
const running_download = running_downloads[i];
const update_obj = {finished_step: true, paused: true, running: false};
if (running_download['step_index'] > 0) {
update_obj['step_index'] = running_download['step_index'] - 1;
}
await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
}
}
async function checkDownloads() {
if (!should_check_downloads) return;
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
await mutex.runExclusive(async () => {
// avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
if (running_downloads.length === 0) {
should_check_downloads = false;
logger.verbose('Disabling checking downloads as none are available.');
}
return;
});
let running_downloads_count = downloads.filter(download => download['running']).length;
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
for (let i = 0; i < waiting_downloads.length; i++) {
const waiting_download = waiting_downloads[i];
const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
if (waiting_download['finished_step'] && !waiting_download['finished']) {
// move to next step
running_downloads_count++;
if (waiting_download['step_index'] === 0) {
collectInfo(waiting_download['uid']);
} else if (waiting_download['step_index'] === 1) {
downloadQueuedFile(waiting_download['uid']);
}
}
}
}
async function collectInfo(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Collecting info for download ${download_uid}`);
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
if (download['user_uid'] && !options.customFileFolderPath) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, download['user_uid'], type);
options.customFileFolderPath = user_path + path.sep;
}
let args = await exports.generateArgs(url, type, options, download['user_uid']);
// get video info prior to download
let info = await getVideoInfoByURL(url, args, download_uid);
if (!info) {
// info failed, error presumably already recorded
return;
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
info = await getVideoInfoByURL(url, args, download_uid);
}
// setup info required to calculate download progress
const expected_file_size = utils.getExpectedFileSize(info);
const files_to_check_for_progress = [];
// store info in download for future use
if (Array.isArray(info)) {
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
} else {
files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
}
const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
finished_step: true,
running: false,
options: options,
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title']
});
}
async function downloadQueuedFile(download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
}
logger.verbose(`Downloading ${download_uid}`);
return new Promise(async resolve => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
const url = download['url'];
const type = download['type'];
const options = download['options'];
const args = download['args'];
const category = download['category'];
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
if (options.customFileFolderPath) {
fileFolderPath = options.customFileFolderPath;
}
fs.ensureDirSync(fileFolderPath);
const start_time = Date.now();
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
// download file
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
const file_objs = [];
let end_time = Date.now();
let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download_uid, err.stderr);
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download_uid, error_message);
logger.warn(error_message);
resolve(false);
return;
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
const ext = type === 'audio' ? '.mp3' : '.mp4';
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj);
}
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download_uid, error_message);
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
resolve();
}
});
});
}
// helper functions
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const globalArgs = config_api.getConfigItem('ytdl_custom_args');
const useCookies = config_api.getConfigItem('ytdl_use_cookies');
const is_audio = type === 'audio';
let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
const customArgs = options.customArgs;
let customOutput = options.customOutput;
const customQualityConfiguration = options.customQualityConfiguration;
// video-specific args
const selectedHeight = options.selectedHeight;
// audio-specific args
const maxBitrate = options.maxBitrate;
const youtubeUsername = options.youtubeUsername;
const youtubePassword = options.youtubePassword;
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
qualityPath = null;
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
qualityPath = ['-f', 'bestvideo+bestaudio']
}
if (customArgs) {
downloadConfig = customArgs.split(',,');
} else {
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
if (customOutput) {
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
if (qualityPath) downloadConfig.push(...qualityPath);
if (is_audio && !options.skip_audio_args) {
downloadConfig.push('-x');
downloadConfig.push('--audio-format', 'mp3');
}
if (youtubeUsername && youtubePassword) {
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
}
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
// if global args has an output, replce the original output with that of global args
const original_output_index = downloadConfig.indexOf('-o');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
}
if (options.additionalArgs && options.additionalArgs !== '') {
downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
}
// filter out incompatible args
downloadConfig = filterArgs(downloadConfig, is_audio);
if (!simulated) logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
return downloadConfig;
}
async function getVideoInfoByURL(url, args = [], download_uid = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
new_args.push('--dump-json');
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
if (output) {
let outputs = [];
try {
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
outputs.push(output_json);
}
resolve(outputs.length === 1 ? outputs[0] : outputs);
} catch(e) {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
await handleDownloadError(download_uid, error);
}
resolve(null);
}
} else {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
await handleDownloadError(download_uid, error_message);
}
resolve(null);
}
});
});
}
function filterArgs(args, isAudio) {
const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
const args_to_remove = isAudio ? video_only_args : audio_only_args;
return args.filter(x => !args_to_remove.includes(x));
}
async function checkDownloadPercent(download_uid) {
/*
This is more of an art than a science, we're just selecting files that start with the file name,
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
be divided by the "total expected bytes."
*/
const download = await db_api.getRecord('download_queue', {uid: download_uid});
const files_to_check_for_progress = download['files_to_check_for_progress'];
const resulting_file_size = download['expected_file_size'];
if (!resulting_file_size) return;
let sum_size = 0;
glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
files.forEach(async file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
});
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
});
}
exports.generateNFOFile = (info, output_path) => {
const nfo_obj = {
episodedetails: {
title: info['fulltitle'],
episode: info['playlist_index'] ? info['playlist_index'] : undefined,
premiered: utils.formatDateString(info['upload_date']),
plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
director: info['artist'] ? info['artist'] : info['uploader']
}
};
const doc = create(nfo_obj);
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join(archivePath);
}
}

View File

@@ -1,8 +0,0 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder",
watch_delay: 5000
}]
}

View File

@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
CMD="pm2-runtime pm2.config.js"
CMD="forever app.js"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then

View File

@@ -1,23 +0,0 @@
const winston = require('winston');
let debugMode = process.env.YTDL_MODE === 'debug';
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: !debugMode ? 'info' : 'debug', name: 'console'})
]
});
module.exports = logger;

View File

@@ -1,6 +1,7 @@
const { app, BrowserWindow } = require('electron');
const path = require('path');
const url = require('url');
const server = require('./app');
let win;
@@ -8,13 +9,7 @@ function createWindow() {
win = new BrowserWindow({ width: 800, height: 600 });
// load the dist folder from Angular
win.loadURL(
url.format({
pathname: path.join(__dirname, `/dist/index.html`),
protocol: 'file:',
slashes: true
})
);
win.loadURL('http://localhost:17442') //ADD THIS
// The following is optional and will open the DevTools:
// win.webContents.openDevTools()

2812
backend/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -2,11 +2,14 @@
"name": "backend",
"version": "1.0.0",
"description": "backend for YoutubeDL-Material",
"main": "index.js",
"main": "main.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
"debug": "set YTDL_MODE=debug && node app.js",
"electron": "electron main.js",
"pack": "electron-builder --dir",
"dist": "electron-builder"
},
"nodemonConfig": {
"ignore": [
@@ -19,6 +22,13 @@
"restart_general.json"
]
},
"build": {
"appId": "youtubedl.material",
"mac": {
"category": "public.app-category.utilities"
},
"files": ["!audio/*", "!video/*", "!users/*", "!subscriptions/*", "!appdata/*"]
},
"repository": {
"type": "git",
"url": ""
@@ -30,14 +40,14 @@
},
"homepage": "",
"dependencies": {
"archiver": "^5.3.1",
"async": "^3.2.3",
"async-mutex": "^0.3.1",
"axios": "^0.21.2",
"archiver": "^3.1.1",
"async": "^3.1.0",
"axios": "^0.21.1",
"bcryptjs": "^2.4.0",
"compression": "^1.7.4",
"config": "^3.2.3",
"express": "^4.17.3",
"exe": "^1.0.2",
"express": "^4.17.1",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
@@ -45,28 +55,29 @@
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^9.2.2",
"moment": "^2.29.2",
"mocha": "^8.4.0",
"moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2",
"node-fetch": "^2.6.7",
"node-fetch": "^2.6.1",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"nodemon": "^2.0.7",
"passport": "^0.5.2",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",
"passport-ldapauth": "^3.0.1",
"passport-ldapauth": "^2.1.4",
"passport-local": "^1.0.0",
"progress": "^2.0.3",
"ps-node": "^0.1.6",
"read-last-lines": "^1.7.2",
"rxjs": "^7.3.0",
"shortid": "^2.2.15",
"unzipper": "^0.10.10",
"uuidv4": "^6.0.6",
"winston": "^3.7.2",
"xmlbuilder2": "^3.0.2",
"winston": "^3.2.1",
"youtube-dl": "^3.0.2"
},
"devDependencies": {
"electron": "^13.1.7",
"electron-builder": "^22.11.7"
}
}

View File

@@ -1,9 +0,0 @@
module.exports = {
apps : [{
name : "YoutubeDL-Material",
script : "./app.js",
watch : "placeholder",
out_file: "/dev/null",
error_file: "/dev/null"
}]
}

View File

@@ -1,15 +1,28 @@
const fs = require('fs-extra');
const path = require('path');
const youtubedl = require('youtube-dl');
const FileSync = require('lowdb/adapters/FileSync')
var fs = require('fs-extra');
const { uuid } = require('uuidv4');
var path = require('path');
var youtubedl = require('youtube-dl');
const config_api = require('./config');
const utils = require('./utils');
const logger = require('./logger');
const twitch_api = require('./twitch');
var utils = require('./utils');
const debugMode = process.env.YTDL_MODE === 'debug';
const db_api = require('./db');
const downloader_api = require('./downloader');
var logger = null;
var db = null;
var users_db = null;
let db_api = null;
function setDB(input_db_api) { db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db_api, input_logger) {
setDB(input_db_api);
setLogger(input_logger);
}
async function subscribe(sub, user_uid = null) {
const result_obj = {
@@ -33,13 +46,13 @@ async function subscribe(sub, user_uid = null) {
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub);
let success = await getSubscriptionInfo(sub, user_uid);
if (success) {
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
}
};
result_obj.success = success;
result_obj.sub = sub;
@@ -48,12 +61,18 @@ async function subscribe(sub, user_uid = null) {
}
async function getSubscriptionInfo(sub) {
async function getSubscriptionInfo(sub, user_uid = null) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
// get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
if (await fs.pathExists(path.join('appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -95,6 +114,22 @@ async function getSubscriptionInfo(sub) {
}
}
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !sub.archive) {
// must create the archive
const archive_dir = path.join(basePath, 'archives', sub.name);
const archive_path = path.join(archive_dir, 'archive.txt');
// creates archive directory and text file if it doesn't exist
fs.ensureDirSync(archive_dir);
fs.ensureFileSync(archive_path);
// updates subscription
sub.archive = archive_dir;
await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
}
// TODO: get even more info
resolve(true);
@@ -111,23 +146,9 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let result_obj = { success: false, error: '' };
let id = sub.id;
const sub_files = await db_api.getRecords('files', {sub_id: id});
for (let i = 0; i < sub_files.length; i++) {
const sub_file = sub_files[i];
if (config_api.descriptors[sub_file['uid']]) {
try {
for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
config_api.descriptors[sub_file['uid']][i].destroy();
}
} catch(e) {
continue;
}
}
}
await db_api.removeRecord('subscriptions', {id: id});
await db_api.removeAllRecords('files', {sub_id: id});
@@ -164,10 +185,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.webp');
var jsonPath = path.join(filePath,name+'.info.json');
var videoFilePath = path.join(filePath,name+ext);
var imageFilePath = path.join(filePath,name+'.jpg');
var altImageFilePath = path.join(filePath,name+'.webp');
const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
fs.pathExists(jsonPath),
@@ -228,15 +249,30 @@ async function getVideosForSub(sub, user_uid = null) {
let appendedBasePath = getAppendedBasePath(sub, basePath);
fs.ensureDirSync(appendedBasePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(async resolve => {
const preimported_file_paths = [];
const PREIMPORT_INTERVAL = 5000;
const preregister_check = setInterval(async () => {
if (sub.streamingOnly) return;
await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
}, PREIMPORT_INTERVAL);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
@@ -244,21 +280,19 @@ async function getVideosForSub(sub, user_uid = null) {
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
// TODO: reimplement
// const outputs = err.stdout.split(/\r\n|\r|\n/);
// for (let i = 0; i < outputs.length; i++) {
// const output = JSON.parse(outputs[i]);
// await handleOutputJSON(sub, output, i === 0, multiUserMode)
// if (err.stderr.includes(output['id']) && archive_path) {
// // we found a video that errored! add it to the archive to prevent future errors
// if (sub.archive) {
// archive_dir = sub.archive;
// archive_path = path.join(archive_dir, 'archive.txt')
// fs.appendFileSync(archive_path, output['id']);
// }
// }
// }
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
await handleOutputJSON(sub, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
fs.appendFileSync(archive_path, output['id']);
}
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
@@ -271,30 +305,21 @@ async function getVideosForSub(sub, user_uid = null) {
resolve(true);
return;
}
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
output_jsons.push(output_json);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const reset_videos = i === 0;
await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
}
resolve(files_to_download);
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
@@ -306,29 +331,10 @@ async function getVideosForSub(sub, user_uid = null) {
}, err => {
logger.error(err);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
});
}
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const base_download_options = {
selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
customFileFolderPath: getAppendedBasePath(sub, basePath),
customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
additionalArgs: sub.custom_args
}
return base_download_options;
}
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
// get basePath
let basePath = null;
@@ -343,14 +349,14 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
let fullOutput = `${appendedBasePath}/${file_output}.%(ext)s`;
if (desired_path) {
fullOutput = `"${desired_path}.%(ext)s"`;
fullOutput = `${desired_path}.%(ext)s`;
} else if (sub.custom_output) {
fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
@@ -365,7 +371,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...qualityPath)
if (sub.custom_args) {
const customArgsArray = sub.custom_args.split(',,');
customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
@@ -396,7 +402,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
if (await fs.pathExists(path.join('appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -407,37 +413,46 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push('--write-thumbnail');
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
return downloadConfig;
}
async function getFilesToDownload(sub, output_jsons) {
const files_to_download = [];
for (let i = 0; i < output_jsons.length; i++) {
const output_json = output_jsons[i];
const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
if (file_missing) {
const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
}
files_to_download.push(output_json);
async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
// TODO: remove streaming only mode
if (false && sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
} else {
path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object);
const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id});
if (file_exists) {
// TODO: fix issue where files of different paths due to custom path get downloaded multiple times
// file already exists in DB, return early to avoid reseting the download date
return;
}
await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);
const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
const file_name = path.basename(output_json['_filename']);
const id = file_name.substring(0, file_name.length-4);
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, id, sub.type, multiUserMode.user, sub);
}
}
return files_to_download;
}
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
@@ -445,7 +460,7 @@ async function getSubscriptions(user_uid = null) {
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
}
async function getSubscription(subID) {
@@ -456,7 +471,7 @@ async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
async function updateSubscription(sub) {
async function updateSubscription(sub, user_uid = null) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
return true;
}
@@ -467,7 +482,7 @@ async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
});
}
async function updateSubscriptionProperty(sub, assignment_obj) {
async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
// TODO: combine with updateSubscription
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
return true;
@@ -522,6 +537,7 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
// helper functions
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
@@ -535,6 +551,7 @@ module.exports = {
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
setLogger : setLogger,
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple
}

View File

@@ -1,195 +0,0 @@
const db_api = require('./db');
const youtubedl_api = require('./youtube-dl');
const fs = require('fs-extra');
const logger = require('./logger');
const scheduler = require('node-schedule');
const TASKS = {
backup_local_db: {
run: db_api.backupDB,
title: 'Backup DB',
job: null
},
missing_files_check: {
run: checkForMissingFiles,
confirm: deleteMissingFiles,
title: 'Missing files check',
job: null
},
missing_db_records: {
run: db_api.importUnregisteredFiles,
title: 'Import missing DB records',
job: null
},
duplicate_files_check: {
run: checkForDuplicateFiles,
confirm: removeDuplicates,
title: 'Find duplicate files in DB',
job: null
},
youtubedl_update_check: {
run: youtubedl_api.checkForYoutubeDLUpdate,
confirm: youtubedl_api.updateYoutubeDL,
title: 'Update youtube-dl',
job: null
}
}
function scheduleJob(task_key, schedule) {
// schedule has to be converted from our format to one node-schedule can consume
let converted_schedule = null;
if (schedule['type'] === 'timestamp') {
converted_schedule = new Date(schedule['data']['timestamp']);
} else if (schedule['type'] === 'recurring') {
const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] ? schedule['data']['minute'] : null;
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
} else {
logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
return null;
}
return scheduler.scheduleJob(converted_schedule, async () => {
const task_state = await db_api.getRecord('tasks', {key: task_key});
if (task_state['running'] || task_state['confirming']) {
logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
return;
}
// remove schedule if it's a one-time task
if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
// we're just "running" the task, any confirmation should be user-initiated
exports.executeRun(task_key);
});
}
if (db_api.database_initialized) {
exports.setupTasks();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) exports.setupTasks();
});
}
exports.setupTasks = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
const task_in_db = await db_api.getRecord('tasks', {key: task_key});
if (!task_in_db) {
// insert task metadata into table if missing
await db_api.insertRecordIntoTable('tasks', {
key: task_key,
title: TASKS[task_key]['title'],
last_ran: null,
last_confirmed: null,
running: false,
confirming: false,
data: null,
error: null,
schedule: null,
options: {}
});
} else {
// reset task if necessary
await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
// schedule task and save job
if (task_in_db['schedule']) {
// prevent timestamp schedules from being set to the past
if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) {
await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
continue;
}
TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
}
}
}
}
exports.executeTask = async (task_key) => {
if (!TASKS[task_key]) {
logger.error(`Task ${task_key} does not exist!`);
return;
}
logger.verbose(`Executing task ${task_key}`);
await exports.executeRun(task_key);
if (!TASKS[task_key]['confirm']) return;
await exports.executeConfirm(task_key);
logger.verbose(`Finished executing ${task_key}`);
}
exports.executeRun = async (task_key) => {
logger.verbose(`Running task ${task_key}`);
// don't set running to true when backup up DB as it will be stick "running" if restored
if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
const data = await TASKS[task_key].run();
await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
logger.verbose(`Finished running task ${task_key}`);
}
exports.executeConfirm = async (task_key) => {
logger.verbose(`Confirming task ${task_key}`);
if (!TASKS[task_key]['confirm']) {
return null;
}
await db_api.updateRecord('tasks', {key: task_key}, {confirming: true});
const task_obj = await db_api.getRecord('tasks', {key: task_key});
const data = task_obj['data'];
await TASKS[task_key].confirm(data);
await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
logger.verbose(`Finished confirming task ${task_key}`);
}
exports.updateTaskSchedule = async (task_key, schedule) => {
logger.verbose(`Updating schedule for task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
if (TASKS[task_key]['job']) {
TASKS[task_key]['job'].cancel();
}
if (schedule) {
TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
}
}
// missing files check
async function checkForMissingFiles() {
const missing_files = [];
const all_files = await db_api.getRecords('files');
for (let i = 0; i < all_files.length; i++) {
const file_to_check = all_files[i];
const file_exists = fs.existsSync(file_to_check['path']);
if (!file_exists) missing_files.push(file_to_check['uid']);
}
return {uids: missing_files};
}
async function deleteMissingFiles(data) {
const uids = data['uids'];
for (let i = 0; i < uids.length; i++) {
const uid = uids[i];
await db_api.removeRecord('files', {uid: uid});
}
}
// duplicate files check
async function checkForDuplicateFiles() {
const duplicate_files = await db_api.findDuplicatesByKey('files', 'path');
const duplicate_uids = duplicate_files.map(duplicate_file => duplicate_file['uid']);
if (duplicate_uids && duplicate_uids.length > 0) {
return {uids: duplicate_uids};
}
return {uids: []};
}
async function removeDuplicates(data) {
for (let i = 0; i < data['uids'].length; i++) {
await db_api.removeRecord('files', {uid: data['uids'][i]});
}
}
exports.TASKS = TASKS;

File diff suppressed because one or more lines are too long

View File

@@ -40,7 +40,7 @@ const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db);
db_api.initialize(db, users_db, logger);
describe('Database', async function() {
@@ -70,17 +70,6 @@ describe('Database', async function() {
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Restore db', async function() {
const db_stats = await db_api.getDBStats();
const file_name = await db_api.backupDB();
await db_api.restoreDB(file_name);
const new_db_stats = await db_api.getDBStats();
assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats));
});
});
describe('Export', function() {
@@ -94,37 +83,12 @@ describe('Database', async function() {
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
@@ -158,7 +122,6 @@ describe('Database', async function() {
});
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
@@ -323,159 +286,5 @@ describe('Multi User', async function() {
// assert(video_obj);
// });
// });
});
describe('Downloader', function() {
const downloader_api = require('../downloader');
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const options = {
ui_uid: uuid(),
user: 'admin'
}
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('download_queue');
});
it('Get file info', async function() {
});
it('Download file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Queue file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
});
it('Pause file', async function() {
});
it('Generate args', async function() {
const args = await downloader_api.generateArgs(url, 'video', options);
console.log(args);
});
it('Generate args - subscription', async function() {
subscriptions_api.initialize(db_api, logger);
const sub = await subscriptions_api.getSubscription(sub_id);
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
const args = await downloader_api.generateArgs(url, 'video', sub_options, 'admin');
console.log(args);
});
it('Generate kodi NFO file', async function() {
const nfo_file_path = './test/sample.nfo';
if (fs.existsSync(nfo_file_path)) {
fs.unlinkSync(nfo_file_path);
}
const sample_json = fs.readJSONSync('./test/sample.info.json');
downloader_api.generateNFOFile(sample_json, nfo_file_path);
assert(fs.existsSync(nfo_file_path), true);
fs.unlinkSync(nfo_file_path);
});
});
describe('Tasks', function() {
const tasks_api = require('../tasks');
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('tasks');
const dummy_task = {
run: async () => { await utils.wait(500); return true; },
confirm: async () => { await utils.wait(500); return true; },
title: 'Dummy task',
job: null
};
tasks_api.TASKS['dummy_task'] = dummy_task;
await tasks_api.initialize();
});
it('Backup db', async function() {
const backups_original = await utils.recFindByExt('appdata', 'bak');
const original_length = backups_original.length;
await tasks_api.executeTask('backup_local_db');
const backups_new = await utils.recFindByExt('appdata', 'bak');
const new_length = backups_new.length;
assert(original_length, new_length-1);
});
it('Check for missing files', async function() {
await db_api.removeAllRecords('files', {uid: 'test'});
const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
await db_api.insertRecordIntoTable('files', test_missing_file);
await tasks_api.executeTask('missing_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
});
it('Check for duplicate files', async function() {
this.timeout(300000);
await db_api.removeAllRecords('files', {uid: 'test1'});
await db_api.removeAllRecords('files', {uid: 'test2'});
const test_duplicate_file1 = {uid: 'test1', path: 'test/missing_file.mp4'};
const test_duplicate_file2 = {uid: 'test2', path: 'test/missing_file.mp4'};
const test_duplicate_file3 = {uid: 'test3', path: 'test/missing_file.mp4'};
await db_api.insertRecordIntoTable('files', test_duplicate_file1);
await db_api.insertRecordIntoTable('files', test_duplicate_file2);
await db_api.insertRecordIntoTable('files', test_duplicate_file3);
await tasks_api.executeTask('duplicate_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
assert(duplicated_record_count == 1, true);
});
it('Import unregistered files', async function() {
this.timeout(300000);
// pre-test cleanup
await db_api.removeAllRecords('files', {title: 'Sample File'});
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
// copies in files
fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
await tasks_api.executeTask('missing_db_records');
const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
assert(!!imported_file, true);
// post-test cleanup
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
});
it('Schedule and cancel task', async function() {
const today_4_hours = new Date();
today_4_hours.setHours(today_4_hours.getHours() + 4);
await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await tasks_api.updateTaskSchedule('dummy_task', null);
assert(!!tasks_api.TASKS['dummy_task']['job'], false);
});
it('Schedule and run task', async function() {
this.timeout(5000);
const today_1_second = new Date();
today_1_second.setSeconds(today_1_second.getSeconds() + 1);
await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await utils.wait(2000);
const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(dummy_task_obj['data'], true);
});
});

View File

@@ -1,13 +1,7 @@
const fs = require('fs-extra');
const path = require('path');
const ffmpeg = require('fluent-ffmpeg');
const archiver = require('archiver');
const fetch = require('node-fetch');
const ProgressBar = require('progress');
const fs = require('fs-extra')
const path = require('path')
const config_api = require('./config');
const logger = require('./logger');
const CONSTS = require('./consts');
const archiver = require('archiver');
const is_windows = process.platform === 'win32';
@@ -48,7 +42,8 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
files.push(jsonobj);
continue;
}
var upload_date = formatDateString(jsonobj.upload_date);
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var isaudio = type === 'audio';
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
@@ -58,13 +53,13 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
return files;
}
async function createContainerZipFile(file_name, container_file_objs) {
async function createContainerZipFile(container_obj, container_file_objs) {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
}
async function createZipFile(zip_file_path, file_paths) {
@@ -146,7 +141,24 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(file_path) {
function getDownloadedThumbnail(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getDownloadedThumbnail2(file_path, type) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
@@ -169,6 +181,10 @@ function getExpectedFileSize(input_info_jsons) {
let expected_filesize = 0;
info_jsons.forEach(info_json => {
if (info_json['filesize']) {
expected_filesize += info_json['filesize'];
return;
}
const formats = info_json['format_id'].split('+');
let individual_expected_filesize = 0;
formats.forEach(format_id => {
@@ -184,7 +200,29 @@ function getExpectedFileSize(input_info_jsons) {
return expected_filesize;
}
function fixVideoMetadataPerms(file_path, type) {
function fixVideoMetadataPerms(name, type, customPath = null) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
path.join(customPath, name + ext + '.info.json'),
// Thumbnails
path.join(customPath, name + '.webp'),
path.join(customPath, name + '.jpg')
];
for (const file of files_to_fix) {
if (!fs.existsSync(file)) continue;
fs.chmodSync(file, 0o644);
}
}
function fixVideoMetadataPerms2(file_path, type) {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
@@ -206,7 +244,19 @@ function fixVideoMetadataPerms(file_path, type) {
}
}
function deleteJSONFile(file_path, type) {
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
let json_path = path.join(customPath, name + '.info.json');
let alternate_json_path = path.join(customPath, name + ext + '.info.json');
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
function deleteJSONFile2(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
@@ -242,6 +292,7 @@ async function removeIDFromArchive(archive_path, id) {
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
function durationStringToNumber(dur_str) {
@@ -264,12 +315,7 @@ function addUIDsToCategory(category, files) {
return files_that_match;
}
function getCurrentDownloader() {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
return details_json['downloader'];
}
async function recFindByExt(base, ext, files, result, recursive = true)
async function recFindByExt(base,ext,files,result)
{
files = files || (await fs.readdir(base))
result = result || []
@@ -278,7 +324,6 @@ async function recFindByExt(base, ext, files, result, recursive = true)
var newbase = path.join(base,file)
if ( (await fs.stat(newbase)).isDirectory() )
{
if (!recursive) continue;
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
}
else
@@ -298,57 +343,6 @@ function removeFileExtension(filename) {
return filename_parts.join('.');
}
function formatDateString(date_string) {
return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
}
function createEdgeNGrams(str) {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
return str.split(" ").reduce((ngrams, token) => {
if (token.length > minGram) {
for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
ngrams = [...ngrams, token.substr(0, i)]
}
} else {
ngrams = [...ngrams, token]
}
return ngrams
}, []).join(" ")
}
return str
}
// ffmpeg helper functions
async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
let base_ffmpeg_call = ffmpeg(file_path);
if (start) {
base_ffmpeg_call = base_ffmpeg_call.seekOutput(start);
}
if (end) {
base_ffmpeg_call = base_ffmpeg_call.duration(end - start);
}
base_ffmpeg_call
.on('end', () => {
logger.verbose(`Cropping for '${file_path}' complete.`);
fs.unlinkSync(file_path);
fs.moveSync(temp_file_path, file_path);
resolve(true);
})
.on('error', (err) => {
logger.error(`Failed to crop ${file_path}.`);
logger.error(err);
resolve(false);
}).save(temp_file_path);
});
}
/**
* setTimeout, but its a promise.
* @param {number} ms
@@ -359,62 +353,6 @@ async function cropFile(file_path, start, end, ext) {
});
}
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
if (watcher) watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
});
}
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
len = parseInt(res.headers.get("Content-Length"), 10);
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();
});
});
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -440,23 +378,20 @@ module.exports = {
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getDownloadedThumbnail2: getDownloadedThumbnail2,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
fixVideoMetadataPerms2: fixVideoMetadataPerms2,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive: removeIDFromArchive,
deleteJSONFile2: deleteJSONFile2,
removeIDFromArchive, removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
checkExistsWithTimeout: checkExistsWithTimeout,
fetchFile: fetchFile,
File: File
}

View File

@@ -1,127 +0,0 @@
const fs = require('fs-extra');
const fetch = require('node-fetch');
const logger = require('./logger');
const utils = require('./utils');
const CONSTS = require('./consts');
const config_api = require('./config.js');
const is_windows = process.platform === 'win32';
const download_sources = {
'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
'func': downloadLatestYoutubeDLBinary
},
'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
'func': downloadLatestYoutubeDLCBinary
},
'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
'func': downloadLatestYoutubeDLPBinary
}
}
exports.checkForYoutubeDLUpdate = async () => {
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const tags_url = download_sources[default_downloader]['tags_url'];
// get current version
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
if (!current_app_details_exists) {
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
}
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
let current_version = current_app_details['version'];
let current_downloader = current_app_details['downloader'];
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(null);
return;
}
}
// got version, now let's check the latest version from the youtube-dl API
fetch(tags_url, {method: 'Get'})
.then(async res => res.json())
.then(async (json) => {
// check if the versions are different
if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`)
resolve(null);
return;
}
const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
// versions different or different downloader is being used, download new update
resolve(latest_update_version);
} else {
resolve(null);
}
return;
})
.catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`)
logger.error(err);
resolve(null);
return;
});
});
}
exports.updateYoutubeDL = async (latest_update_version) => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await download_sources[default_downloader]['func'](latest_update_version);
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version;
details_json['downloader'] = downloader;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
}

View File

@@ -1,31 +0,0 @@
#!/bin/sh
# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
# and also optimizing some code with this commit.
# xoxo :D
set -xeuo pipefail
case $(uname -m) in
x86_64)
ARCH=amd64;;
aarch64)
ARCH=arm64;;
armhf)
ARCH=armhf;;
armv7)
ARCH=armel;;
armv7l)
ARCH=armel;;
*)
echo "Unsupported architecture: $(uname -m)"
exit 1
esac
echo "Architecture: $ARCH"
wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
mkdir /tmp/ffmpeg
tar xf ffmpeg.txz -C /tmp/ffmpeg
cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
rm -rf /tmp/ffmpeg ffmpeg.txz

View File

@@ -7,8 +7,6 @@ services:
ytdl_use_local_db: 'false'
write_ytdl_config: 'true'
restart: always
depends_on:
- ytdl-mongo-db
volumes:
- ./appdata:/app/appdata
- ./audio:/app/audio
@@ -17,7 +15,7 @@ services:
- ./users:/app/users
ports:
- "8998:17442"
image: tzahi12345/youtubedl-material:nightly
image: tzahi12345/youtubedl-material:latest
ytdl-mongo-db:
image: mongo
ports:
@@ -25,6 +23,5 @@ services:
logging:
driver: "none"
container_name: mongo-db
restart: always
volumes:
- ./db/:/data/db

13397
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -5,14 +5,12 @@
"scripts": {
"ng": "ng",
"start": "ng serve",
"build": "ng build --configuration production",
"prebuild": "node src/postbuild.mjs",
"build": "ng build",
"heroku-postbuild": "npm install --prefix backend",
"test": "ng test",
"lint": "ng lint",
"e2e": "ng e2e",
"electron": "ng build --base-href ./ && electron .",
"generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
"electron": "ng build --base-href ./ && electron ."
},
"engines": {
"node": "12.3.1",
@@ -20,62 +18,55 @@
},
"private": true,
"dependencies": {
"@angular-devkit/core": "^13.3.3",
"@angular/animations": "^13.3.4",
"@angular/cdk": "^13.3.4",
"@angular/common": "^13.3.4",
"@angular/compiler": "^13.3.4",
"@angular/core": "^13.3.4",
"@angular/forms": "^13.3.4",
"@angular/localize": "^13.3.4",
"@angular/material": "^13.3.4",
"@angular/platform-browser": "^13.3.4",
"@angular/platform-browser-dynamic": "^13.3.4",
"@angular/router": "^13.3.4",
"@fontsource/material-icons": "^4.5.4",
"@angular-devkit/core": "^11.0.4",
"@angular/animations": "^11.0.4",
"@angular/cdk": "^11.0.2",
"@angular/common": "^11.0.4",
"@angular/compiler": "^11.0.4",
"@angular/core": "^11.0.4",
"@angular/forms": "^11.0.4",
"@angular/localize": "^11.0.4",
"@angular/material": "^11.0.2",
"@angular/platform-browser": "^11.0.4",
"@angular/platform-browser-dynamic": "^11.0.4",
"@angular/router": "^11.0.4",
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^5.0.1",
"@videogular/ngx-videogular": "^2.1.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
"filesize": "^6.1.0",
"fingerprintjs2": "^2.1.0",
"fs-extra": "^10.0.0",
"material-icons": "^1.10.8",
"material-icons": "^0.5.4",
"nan": "^2.14.1",
"ng-lazyload-image": "^7.0.1",
"ngx-avatars": "^1.3.1",
"ngx-file-drop": "^13.0.0",
"ngx-avatar": "^4.0.0",
"ngx-file-drop": "^9.0.1",
"rxjs": "^6.6.3",
"rxjs-compat": "^6.0.0-rc.0",
"tslib": "^2.0.0",
"typescript": "~4.6.3",
"xliff-to-json": "^1.0.4",
"zone.js": "~0.11.4"
"typescript": "~4.0.5",
"web-animations-js": "^2.3.2",
"zone.js": "~0.10.2"
},
"devDependencies": {
"@angular-devkit/build-angular": "^13.3.3",
"@angular/cli": "^13.3.3",
"@angular/compiler-cli": "^13.3.4",
"@angular/language-service": "^13.3.4",
"@angular-devkit/build-angular": "^0.1100.4",
"@angular/cli": "^11.0.4",
"@angular/compiler-cli": "^11.0.4",
"@angular/language-service": "^11.0.4",
"@types/core-js": "^2.5.2",
"@types/file-saver": "^2.0.1",
"@types/jasmine": "~3.6.0",
"@types/node": "^12.11.1",
"@typescript-eslint/eslint-plugin": "^4.29.0",
"@typescript-eslint/parser": "^4.29.0",
"codelyzer": "^6.0.0",
"electron": "^13.6.6",
"eslint": "^7.32.0",
"electron": "^8.0.1",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
"karma": "~6.3.16",
"karma": "~5.0.0",
"karma-chrome-launcher": "~3.1.0",
"karma-cli": "~1.0.1",
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"openapi-typescript-codegen": "^0.21.0",
"protractor": "~7.0.0",
"ts-node": "~3.0.4",
"tslint": "~6.1.0"

View File

@@ -1,11 +1,11 @@
/* Coolors Exported Palette - coolors.co/e8aeb7-b8e1ff-a9fff7-94fbab-82aba1 */
/* HSL */
$color1: hsla(351, 56%, 80%, 1);
$softblue: hsla(205, 100%, 86%, 1);
$color3: hsla(174, 100%, 83%, 1);
$color4: hsla(133, 93%, 78%, 1);
$color5: hsla(165, 20%, 59%, 1);
$color1: hsla(351%, 56%, 80%, 1);
$softblue: hsla(205%, 100%, 86%, 1);
$color3: hsla(174%, 100%, 83%, 1);
$color4: hsla(133%, 93%, 78%, 1);
$color5: hsla(165%, 20%, 59%, 1);
/* RGB */
$color1: rgba(232, 174, 183, 1);

View File

@@ -1,111 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
export type { body_19 } from './models/body_19';
export type { body_20 } from './models/body_20';
export type { Category } from './models/Category';
export { CategoryRule } from './models/CategoryRule';
export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermissionsRequest';
export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest';
export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest';
export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse';
export type { ConcurrentStream } from './models/ConcurrentStream';
export type { Config } from './models/Config';
export type { ConfigResponse } from './models/ConfigResponse';
export type { CreateCategoryRequest } from './models/CreateCategoryRequest';
export type { CreateCategoryResponse } from './models/CreateCategoryResponse';
export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
export type { CropFileSettings } from './models/CropFileSettings';
export type { DatabaseFile } from './models/DatabaseFile';
export { DBBackup } from './models/DBBackup';
export type { DBInfoResponse } from './models/DBInfoResponse';
export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
export type { DeleteUserRequest } from './models/DeleteUserRequest';
export type { Download } from './models/Download';
export type { DownloadArchiveRequest } from './models/DownloadArchiveRequest';
export type { DownloadFileRequest } from './models/DownloadFileRequest';
export type { DownloadRequest } from './models/DownloadRequest';
export type { DownloadResponse } from './models/DownloadResponse';
export type { DownloadTwitchChatByVODIDRequest } from './models/DownloadTwitchChatByVODIDRequest';
export type { DownloadTwitchChatByVODIDResponse } from './models/DownloadTwitchChatByVODIDResponse';
export type { DownloadVideosForSubscriptionRequest } from './models/DownloadVideosForSubscriptionRequest';
export type { File } from './models/File';
export { FileType } from './models/FileType';
export type { GenerateArgsResponse } from './models/GenerateArgsResponse';
export type { GenerateNewApiKeyResponse } from './models/GenerateNewApiKeyResponse';
export type { GetAllCategoriesResponse } from './models/GetAllCategoriesResponse';
export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
export type { GetDownloadRequest } from './models/GetDownloadRequest';
export type { GetDownloadResponse } from './models/GetDownloadResponse';
export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
export type { GetFileFormatsResponse } from './models/GetFileFormatsResponse';
export type { GetFileRequest } from './models/GetFileRequest';
export type { GetFileResponse } from './models/GetFileResponse';
export type { GetFullTwitchChatRequest } from './models/GetFullTwitchChatRequest';
export type { GetFullTwitchChatResponse } from './models/GetFullTwitchChatResponse';
export type { GetLogsRequest } from './models/GetLogsRequest';
export type { GetLogsResponse } from './models/GetLogsResponse';
export type { GetMp3sResponse } from './models/GetMp3sResponse';
export type { GetMp4sResponse } from './models/GetMp4sResponse';
export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
export type { GetRolesResponse } from './models/GetRolesResponse';
export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
export type { GetTaskRequest } from './models/GetTaskRequest';
export type { GetTaskResponse } from './models/GetTaskResponse';
export type { GetUsersResponse } from './models/GetUsersResponse';
export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
export type { inline_response_200_15 } from './models/inline_response_200_15';
export type { LoginRequest } from './models/LoginRequest';
export type { LoginResponse } from './models/LoginResponse';
export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse';
export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
export { Schedule } from './models/Schedule';
export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SharingToggle } from './models/SharingToggle';
export type { SubscribeRequest } from './models/SubscribeRequest';
export type { SubscribeResponse } from './models/SubscribeResponse';
export type { Subscription } from './models/Subscription';
export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
export type { SuccessObject } from './models/SuccessObject';
export type { TableInfo } from './models/TableInfo';
export type { Task } from './models/Task';
export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
export type { TransferDBRequest } from './models/TransferDBRequest';
export type { TransferDBResponse } from './models/TransferDBResponse';
export type { TwitchChatMessage } from './models/TwitchChatMessage';
export type { UnsubscribeRequest } from './models/UnsubscribeRequest';
export type { UnsubscribeResponse } from './models/UnsubscribeResponse';
export type { UpdateCategoriesRequest } from './models/UpdateCategoriesRequest';
export type { UpdateCategoryRequest } from './models/UpdateCategoryRequest';
export type { UpdateConcurrentStreamRequest } from './models/UpdateConcurrentStreamRequest';
export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentStreamResponse';
export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
export type { UpdaterStatus } from './models/UpdaterStatus';
export type { UpdateServerRequest } from './models/UpdateServerRequest';
export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
export type { UpdateUserRequest } from './models/UpdateUserRequest';
export type { User } from './models/User';
export { UserPermission } from './models/UserPermission';
export type { Version } from './models/Version';
export type { VersionInfoResponse } from './models/VersionInfoResponse';
export { YesNo } from './models/YesNo';

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface AddFileToPlaylistRequest {
file_uid: string;
playlist_id: string;
}

View File

@@ -1,11 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { UserPermission } from './UserPermission';
import { YesNo } from './YesNo';
export interface BaseChangePermissionsRequest {
permission: UserPermission;
new_value: YesNo;
}

View File

@@ -1,15 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { CategoryRule } from './CategoryRule';
export interface Category {
name?: string;
uid?: string;
rules?: Array<CategoryRule>;
/**
* Overrides file output for downloaded files in category
*/
custom_output?: string;
}

View File

@@ -1,26 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CategoryRule {
preceding_operator?: CategoryRule.preceding_operator;
comparator?: CategoryRule.comparator;
}
export namespace CategoryRule {
export enum preceding_operator {
OR = 'or',
AND = 'and',
}
export enum comparator {
INCLUDES = 'includes',
NOT_INCLUDES = 'not_includes',
EQUALS = 'equals',
NOT_EQUALS = 'not_equals',
}
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
export interface ChangeRolePermissionsRequest extends BaseChangePermissionsRequest {
role: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { BaseChangePermissionsRequest } from './BaseChangePermissionsRequest';
export interface ChangeUserPermissionsRequest extends BaseChangePermissionsRequest {
user_uid: string;
}

View File

@@ -1,11 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CheckConcurrentStreamRequest {
/**
* UID of the concurrent stream
*/
uid: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { ConcurrentStream } from './ConcurrentStream';
export interface CheckConcurrentStreamResponse {
stream: ConcurrentStream;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface ConcurrentStream {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Config {
YoutubeDLMaterial: any;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Config } from './Config';
export interface ConfigResponse {
config_file: Config;
success: boolean;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CreateCategoryRequest {
name: string;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Category } from './Category';
export interface CreateCategoryResponse {
new_category?: Category;
success?: boolean;
}

View File

@@ -1,12 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface CreatePlaylistRequest {
playlistName: string;
uids: Array<string>;
type: FileType;
thumbnailURL: string;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Playlist } from './Playlist';
export interface CreatePlaylistResponse {
new_playlist: Playlist;
success: boolean;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface CropFileSettings {
cropFileStart: number;
cropFileEnd: number;
}

View File

@@ -1,21 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DBBackup {
name: string;
timestamp: number;
size: number;
source: DBBackup.source;
}
export namespace DBBackup {
export enum source {
LOCAL = 'local',
REMOTE = 'remote',
}
}

View File

@@ -1,18 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { TableInfo } from './TableInfo';
export interface DBInfoResponse {
using_local_db?: boolean;
stats_by_table?: {
files?: TableInfo,
playlists?: TableInfo,
categories?: TableInfo,
subscriptions?: TableInfo,
users?: TableInfo,
roles?: TableInfo,
download_queue?: TableInfo,
};
}

View File

@@ -1,22 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DatabaseFile {
id: string;
title: string;
thumbnailURL: string;
isAudio: boolean;
/**
* In seconds
*/
duration: number;
url: string;
uploader: string;
size: number;
path: string;
upload_date: string;
uid: string;
sharingEnabled?: boolean;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteCategoryRequest {
category_uid: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteMp3Mp4Request {
uid: string;
blacklistMode?: boolean;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface DeletePlaylistRequest {
playlist_id: string;
type: FileType;
}

View File

@@ -1,15 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { SubscriptionRequestData } from './SubscriptionRequestData';
export interface DeleteSubscriptionFileRequest {
file: string;
file_uid?: string;
sub: SubscriptionRequestData;
/**
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DeleteUserRequest {
uid: string;
}

View File

@@ -1,7 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type Dictionary<T> = {
[key: string]: T;
}

View File

@@ -1,26 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Download {
uid: string;
ui_uid?: string;
running: boolean;
finished: boolean;
paused: boolean;
finished_step: boolean;
url: string;
type: string;
title: string;
step_index: number;
percent_complete: number;
timestamp_start: number;
/**
* Error text, set if download fails.
*/
error?: string | null;
user_uid?: string;
sub_id?: string;
sub_name?: string;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DownloadArchiveRequest {
sub: {
archive_dir: string,
};
}

View File

@@ -1,14 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface DownloadFileRequest {
uid?: string;
uuid?: string;
sub_id?: string;
playlist_id?: string;
url?: string;
type?: FileType;
}

View File

@@ -1,44 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { CropFileSettings } from './CropFileSettings';
import { FileType } from './FileType';
export interface DownloadRequest {
url: string;
/**
* Video format code. Overrides other quality options.
*/
customQualityConfiguration?: string;
/**
* Custom command-line arguments for youtube-dl. Overrides all other options, except url.
*/
customArgs?: string;
/**
* Additional command-line arguments for youtube-dl. Added to whatever args would normally be used.
*/
additionalArgs?: string;
/**
* Custom output filename template.
*/
customOutput?: string;
/**
* Login with this account ID
*/
youtubeUsername?: string;
/**
* Account password
*/
youtubePassword?: string;
/**
* Height of the video, if known
*/
selectedHeight?: string;
/**
* Specify ffmpeg/avconv audio quality
*/
maxBitrate?: string;
type?: FileType;
cropFileSettings?: CropFileSettings;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface DownloadResponse {
download?: Download;
}

View File

@@ -1,23 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
import { Subscription } from './Subscription';
export interface DownloadTwitchChatByVODIDRequest {
/**
* File ID
*/
id: string;
/**
* ID of the VOD
*/
vodId: string;
type: FileType;
/**
* User UID
*/
uuid?: string;
sub?: Subscription;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { TwitchChatMessage } from './TwitchChatMessage';
export interface DownloadTwitchChatByVODIDResponse {
chat: Array<TwitchChatMessage>;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DownloadVideosForSubscriptionRequest {
subID: string;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface File {
id?: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GenerateArgsResponse {
args?: Array<string>;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GenerateNewApiKeyResponse {
new_api_key: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Category } from './Category';
export interface GetAllCategoriesResponse {
categories: Array<Category>;
}

View File

@@ -1,11 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetAllDownloadsRequest {
/**
* Filters downloads with the array
*/
uids?: Array<string> | null;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface GetAllDownloadsResponse {
downloads?: Array<Download>;
}

View File

@@ -1,14 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
import { Playlist } from './Playlist';
export interface GetAllFilesResponse {
files: Array<DatabaseFile>;
/**
* All video playlists
*/
playlists: Array<Playlist>;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Subscription } from './Subscription';
export interface GetAllSubscriptionsResponse {
subscriptions: Array<Subscription>;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Task } from './Task';
export interface GetAllTasksResponse {
tasks?: Array<Task>;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DBBackup } from './DBBackup';
export interface GetDBBackupsResponse {
tasks?: Array<DBBackup>;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetDownloadRequest {
download_uid: string;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Download } from './Download';
export interface GetDownloadResponse {
download?: Download;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetFileFormatsRequest {
url?: string;
}

View File

@@ -1,12 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { File } from './File';
export interface GetFileFormatsResponse {
success: boolean;
result: {
formats?: Array<any>,
};
}

View File

@@ -1,17 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface GetFileRequest {
/**
* Video UID
*/
uid: string;
type?: FileType;
/**
* User UID
*/
uuid?: string;
}

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
export interface GetFileResponse {
success: boolean;
file?: DatabaseFile;
}

View File

@@ -1,19 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
import { Subscription } from './Subscription';
export interface GetFullTwitchChatRequest {
/**
* File ID
*/
id: string;
type: FileType;
/**
* User UID
*/
uuid?: string;
sub?: Subscription;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetFullTwitchChatResponse {
success: boolean;
error?: string;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetLogsRequest {
lines?: number;
}

View File

@@ -1,12 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetLogsResponse {
/**
* Number of lines to retrieve from the bottom
*/
logs?: string;
success?: boolean;
}

View File

@@ -1,14 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
import { Playlist } from './Playlist';
export interface GetMp3sResponse {
mp3s: Array<DatabaseFile>;
/**
* All audio playlists
*/
playlists: Array<Playlist>;
}

View File

@@ -1,14 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DatabaseFile } from './DatabaseFile';
import { Playlist } from './Playlist';
export interface GetMp4sResponse {
mp4s: Array<DatabaseFile>;
/**
* All video playlists
*/
playlists: Array<Playlist>;
}

View File

@@ -1,12 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
export interface GetPlaylistRequest {
playlist_id: string;
type?: FileType;
uuid?: string;
include_file_metadata?: boolean;
}

View File

@@ -1,12 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { FileType } from './FileType';
import { Playlist } from './Playlist';
export interface GetPlaylistResponse {
playlist: Playlist;
type: FileType;
success: boolean;
}

View File

@@ -1,8 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetPlaylistsRequest {
include_categories?: boolean;
}

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Playlist } from './Playlist';
export interface GetPlaylistsResponse {
playlists: Array<Playlist>;
}

View File

@@ -1,16 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { UserPermission } from './UserPermission';
export interface GetRolesResponse {
roles: {
admin?: {
permissions?: Array<UserPermission>,
},
user?: {
permissions?: Array<UserPermission>,
},
};
}

Some files were not shown because too many files have changed in this diff Show More