Compare commits

..

1 Commits

Author SHA1 Message Date
Isaac Abadi
3e3a552392 Updated Angular to v14 2022-07-06 01:19:07 -04:00
292 changed files with 5702 additions and 40083 deletions

View File

@@ -17,7 +17,7 @@ jobs:
- name: setup node
uses: actions/setup-node@v2
with:
node-version: '14'
node-version: '12'
cache: 'npm'
- name: install dependencies
run: |

View File

@@ -1,11 +0,0 @@
{
"recommendations": [
"angular.ng-template",
"dbaeumer.vscode-eslint",
"waderyan.gitblame",
"42crunch.vscode-openapi",
"redhat.vscode-yaml",
"christian-kohler.npm-intellisense",
"hbenl.vscode-mocha-test-adapter"
]
}

View File

@@ -1,8 +0,0 @@
{
"mochaExplorer.files": "backend/test/**/*.js",
"mochaExplorer.cwd": "backend",
"mochaExplorer.globImplementation": "vscode",
"mochaExplorer.env": {
"YTDL_MODE": "debug"
}
}

45
.vscode/tasks.json vendored
View File

@@ -1,60 +1,25 @@
{
"version": "2.0.0",
"windows": {
"options": {
"shell": {
"executable": "cmd.exe",
"args": [
"/d", "/c"
]
}
}
},
"tasks": [
{
"type": "npm",
"script": "start",
"problemMatcher": [],
"label": "Dev: start frontend",
"detail": "ng serve",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": true,
"clear": false
}
"detail": "ng serve"
},
{
"label": "Dev: start backend",
"type": "shell",
"command": "node app.js",
"command": "set YTDL_MODE=debug && node app.js",
"options": {
"cwd": "./backend",
"env": {
"YTDL_MODE": "debug"
}
"cwd": "./backend"
},
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": true,
"clear": false
"panel": "new"
},
"problemMatcher": [],
"dependsOn": ["Dev: post-build"]
},
{
"label": "Dev: post-build",
"type": "shell",
"command": "node src/postbuild.mjs"
},
{
"label": "Dev: run all",
"dependsOn": ["Dev: start backend", "Dev: start frontend"]
"problemMatcher": []
}
]
}

View File

@@ -2,12 +2,11 @@
FROM ubuntu:22.04 AS ffmpeg
ENV DEBIAN_FRONTEND=noninteractive
# Use script due local build compability
COPY docker-utils/ffmpeg-fetch.sh .
RUN chmod +x ffmpeg-fetch.sh
COPY ffmpeg-fetch.sh .
RUN sh ./ffmpeg-fetch.sh
# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
# Create our Ubuntu 22.04 with node 16
# Go to 20.04
FROM ubuntu:20.04 AS base
ARG DEBIAN_FRONTEND=noninteractive
@@ -22,8 +21,7 @@ RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
apt install -y --no-install-recommends curl ca-certificates tzdata && \
curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
apt install -y --no-install-recommends nodejs && \
npm -g install npm n && \
n 16.14.2 && \
npm -g install npm && \
apt clean && \
rm -rf /var/lib/apt/lists/*
@@ -47,31 +45,21 @@ RUN npm config set strict-ssl false && \
npm install --prod && \
ls -al
FROM base as python
WORKDIR /app
COPY docker-utils/GetTwitchDownloader.py .
RUN apt update && \
apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install PyGithub requests
RUN python GetTwitchDownloader.py
# Final image
FROM base
RUN npm install -g pm2 && \
apt update && \
apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install pycryptodomex
RUN pip install tcd
WORKDIR /app
# User 1000 already exist from base image
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
RUN chown $UID:$GID .
RUN chmod +x /app/fix-scripts/*.sh
# Add some persistence data
#VOLUME ["/app/appdata"]

View File

@@ -111,37 +111,6 @@ paths:
$ref: '#/components/schemas/GetAllFilesResponse'
security:
- Auth query parameter: []
/api/rss:
get:
tags:
- files
summary: Generates an RSS feed
description: Generates an RSS feed for downloaded files
operationId: get-rss
parameters:
- in: query
name: params
schema:
allOf:
- $ref: '#/components/schemas/GetAllFilesRequest'
- type: object
properties:
uuid:
type: string
description: user uid
default: null
style: form
explode: true
responses:
'200':
description: OK
content:
text/plain:
schema:
type: string
description: RSS feed
security:
- Auth query parameter: []
/api/getFile:
post:
tags:
@@ -578,69 +547,6 @@ paths:
description: If the archive dir is not found, 404 is sent as a response
security:
- Auth query parameter: []
/api/deleteArchiveItems:
post:
tags:
- archive
summary: Delete item from archive
description: 'Deletes an item from the archive'
operationId: post-api-deleteArchiveItems
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/DeleteArchiveItemsRequest'
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/importArchive:
post:
tags:
- archive
summary: Imports archive
description: 'Imports an existing archive.txt file'
operationId: post-api-importArchive
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/ImportArchiveRequest'
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/uploadCookies:
post:
tags:
- downloader
summary: Upload cookies
description: 'Uploads cookies file to be used during downloading'
operationId: post-api-uploadCookies
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/UploadCookiesRequest'
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/updaterStatus:
get:
tags:
@@ -678,6 +584,22 @@ paths:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/isPinSet:
post:
tags:
- security
summary: Check if pin is set
description: Checks if the pin is set for settings
operationId: post-api-isPinSet
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/inline_response_200_15'
security:
- Auth query parameter: []
/api/generateNewAPIKey:
post:
tags:
@@ -889,7 +811,7 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/RestartDownloadResponse'
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
@@ -1295,48 +1217,6 @@ paths:
- Auth query parameter: []
tags:
- multi-user mode
/api/setPin:
post:
summary: Set settings pin
operationId: post-api-setPin
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/SetPinRequest'
description: 'Sets a pin for the settings'
security:
- Auth query parameter: []
tags:
- security
/api/auth/pinLogin:
post:
summary: Pin login
operationId: post-api-pin-login
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/PinLoginResponse'
description: Use this endpoint to generate a JWT token for pin authentication. Put anything in the username field.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/LoginRequest'
security:
- Auth query parameter: []
tags:
- security
/api/getUsers:
post:
summary: Get all users
@@ -1709,10 +1589,6 @@ components:
type: string
description: Height of the video, if known
example: '1080'
maxHeight:
type: string
description: Max height that should be used, useful for playlists. selectedHeight will override this.
example: '1080'
maxBitrate:
type: string
description: Specify ffmpeg/avconv audio quality
@@ -1721,9 +1597,6 @@ components:
$ref: '#/components/schemas/FileType'
cropFileSettings:
$ref: '#/components/schemas/CropFileSettings'
ignoreArchive:
type: boolean
description: If using youtube-dl archive, download will ignore it
DownloadResponse:
type: object
properties:
@@ -1748,13 +1621,6 @@ components:
properties:
download:
$ref: '#/components/schemas/Download'
RestartDownloadResponse:
allOf:
- $ref: '#/components/schemas/SuccessObject'
- type: object
properties:
new_download_uid:
type: string
GetAllDownloadsRequest:
type: object
properties:
@@ -1807,16 +1673,6 @@ components:
required:
- task_key
- new_data
UpdateTaskOptionsRequest:
type: object
properties:
task_key:
type: string
new_options:
type: object
required:
- task_key
- new_options
GetTaskResponse:
type: object
properties:
@@ -1885,39 +1741,29 @@ components:
description: Two elements allowed, start index and end index
minItems: 2
maxItems: 2
default: null
text_search:
type: string
description: Filter files by title
default: null
file_type_filter:
$ref: '#/components/schemas/FileTypeFilter'
favorite_filter:
type: boolean
description: If set to true, only gets favorites
default: false
sub_id:
type: string
description: Include if you want to filter by subscription
default: null
Sort:
type: object
properties:
by:
type: string
description: Property to sort by
default: registered
order:
type: number
description: 1 for ascending, -1 for descending
default: -1
FileTypeFilter:
type: string
enum:
- audio_only
- video_only
- both
default: both
GetAllFilesResponse:
required:
- files
@@ -2035,11 +1881,16 @@ components:
description: Number of files removed
DeleteSubscriptionFileRequest:
required:
- file_uid
- file
- sub
type: object
properties:
file:
type: string
file_uid:
type: string
sub:
$ref: '#/components/schemas/SubscriptionRequestData'
deleteForever:
type: boolean
description: 'If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.'
@@ -2179,83 +2030,17 @@ components:
type:
$ref: '#/components/schemas/FileType'
DownloadArchiveRequest:
type: object
properties:
type:
$ref: '#/components/schemas/FileType'
sub_id:
type: string
Archive:
required:
- extractor
- id
- type
- title
- timestamp
- uid
- sub
type: object
properties:
extractor:
type: string
id:
type: string
type:
$ref: '#/components/schemas/FileType'
title:
type: string
user_uid:
type: string
sub_id:
type: string
timestamp:
type: number
uid:
type: string
DeleteArchiveItemsRequest:
type: object
required:
- archives
properties:
archives:
type: array
items:
$ref: '#/components/schemas/Archive'
ImportArchiveRequest:
type: object
required:
- archive
- type
properties:
archive:
type: string
type:
$ref: '#/components/schemas/FileType'
sub_id:
type: string
GetArchivesRequest:
type: object
properties:
type:
$ref: '#/components/schemas/FileType'
sub_id:
type: string
GetArchivesResponse:
type: object
required:
- archives
properties:
archives:
type: array
items:
$ref: '#/components/schemas/Archive'
UploadCookiesRequest:
type: object
required:
- cookies
properties:
cookies:
type: string
format: binary
sub:
required:
- archive_dir
type: object
properties:
archive_dir:
type: string
UpdaterStatus:
required:
- details
@@ -2276,6 +2061,8 @@ components:
tag:
type: string
DBInfoResponse:
required:
- db_info
type: object
properties:
using_local_db:
@@ -2297,8 +2084,6 @@ components:
$ref: '#/components/schemas/TableInfo'
download_queue:
$ref: '#/components/schemas/TableInfo'
archives:
$ref: '#/components/schemas/TableInfo'
TransferDBResponse:
required:
- success
@@ -2598,7 +2383,6 @@ components:
- upload_date
- uploader
- url
- favorite
type: object
properties:
id:
@@ -2628,8 +2412,6 @@ components:
type: string
uid:
type: string
user_uid:
type: string
sharingEnabled:
type: boolean
category:
@@ -2648,8 +2430,6 @@ components:
abr:
type: number
description: In Kbps
favorite:
type: boolean
Playlist:
required:
- uids
@@ -2681,8 +2461,6 @@ components:
type: string
auto:
type: boolean
sharingEnabled:
type: boolean
Download:
required:
- url
@@ -2727,10 +2505,6 @@ components:
type: string
description: Error text, set if download fails.
nullable: true
error_type:
type: string
description: Error type, may or may not be set in case of an error
nullable: true
user_uid:
type: string
sub_id:
@@ -2769,8 +2543,6 @@ components:
type: string
schedule:
type: object
options:
type: object
Schedule:
required:
- type
@@ -2795,8 +2567,6 @@ components:
type: number
timestamp:
type: number
tz:
type: string
DBBackup:
required:
- name
@@ -2985,44 +2755,6 @@ components:
type: string
date:
type: string
Notification:
required:
- uid
- type
- text
- read
- timestamp
type: object
properties:
type:
$ref: '#/components/schemas/NotificationType'
uid:
type: string
user_uid:
type: string
action:
type: array
items:
$ref: '#/components/schemas/NotificationAction'
read:
type: boolean
data:
type: object
timestamp:
type: number
NotificationAction:
type: string
enum:
- play
- retry_download
- view_download_error
- view_tasks
NotificationType:
type: string
enum:
- download_complete
- download_error
- task_finished
BaseChangePermissionsRequest:
required:
- permission
@@ -3051,13 +2783,6 @@ components:
type: string
required:
- role
SetPinRequest:
required:
- new_pin
type: object
properties:
new_pin:
type: string
file:
title: file
type: object
@@ -3107,13 +2832,6 @@ components:
type: array
items:
$ref: '#/components/schemas/UserPermission'
PinLoginResponse:
required:
- pin_token
type: object
properties:
pin_token:
type: string
UpdateUserRequest:
required:
- change_object
@@ -3168,29 +2886,6 @@ components:
type: array
items:
$ref: '#/components/schemas/UserPermission'
DeleteNotificationRequest:
required:
- uid
type: object
properties:
uid:
type: string
SetNotificationsToReadRequest:
required:
- uids
type: object
properties:
uids:
type: array
items:
type: string
GetNotificationsResponse:
type: object
properties:
notifications:
type: array
items:
$ref: '#/components/schemas/Notification'
securitySchemes:
Auth query parameter:
name: apiKey

View File

@@ -6,7 +6,7 @@
[![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
[![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 15](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
Now with [Docker](#Docker) support!
@@ -14,7 +14,7 @@ Now with [Docker](#Docker) support!
## Getting Started
Check out the prerequisites, and go to the [installation](#Installing) section. Easy as pie!
Check out the prerequisites, and go to the installation section. Easy as pie!
Here's an image of what it'll look like once you're done:
@@ -52,8 +52,6 @@ Optional dependencies:
### Installing
If you are using Docker, skip to the [Docker](#Docker) section. Otherwise, continue:
1. First, download the [latest release](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest)!
2. Drag the `youtubedl-material` directory to an easily accessible directory. Navigate to the `appdata` folder and edit the `default.json` file.

View File

@@ -190,8 +190,5 @@
"@schematics/angular:directive": {
"prefix": "app"
}
},
"cli": {
"analytics": false
}
}

View File

@@ -18,7 +18,6 @@ const URL = require('url').URL;
const CONSTS = require('./consts')
const read_last_lines = require('read-last-lines');
const ps = require('ps-node');
const Feed = require('feed').Feed;
// needed if bin/details somehow gets deleted
if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -33,7 +32,6 @@ const subscriptions_api = require('./subscriptions');
const categories_api = require('./categories');
const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
var app = express();
@@ -71,8 +69,7 @@ db.defaults(
downloads: {},
subscriptions: [],
files_to_db_migration_complete: false,
tasks_manager_role_migration_complete: false,
archives_migration_complete: false
tasks_manager_role_migration_complete: false
}).write();
users_db.defaults(
@@ -162,7 +159,6 @@ app.use(bodyParser.json());
// use passport
app.use(auth_api.passport.initialize());
app.use(auth_api.passport.session());
// actual functions
@@ -203,15 +199,6 @@ async function checkMigrations() {
db.set('tasks_manager_role_migration_complete', true).write();
}
const archives_migration_complete = db.get('archives_migration_complete').value();
if (!archives_migration_complete) {
logger.info('Checking if archives have been migrated...');
const imported_archives = await archive_api.importArchives();
if (imported_archives) logger.info('Archives migration complete!');
else logger.error('Failed to migrate archives!');
db.set('archives_migration_complete', true).write();
}
return true;
}
@@ -523,6 +510,9 @@ async function loadConfig() {
db_api.database_initialized = true;
db_api.database_initialized_bs.next(true);
// creates archive path if missing
await fs.ensureDir(utils.getArchiveFolder());
// check migrations
await checkMigrations();
@@ -568,7 +558,14 @@ function loadConfigValues() {
url_domain = new URL(url);
let logger_level = config_api.getConfigItem('ytdl_logger_level');
utils.updateLoggerLevel(logger_level);
const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
if (!possible_levels.includes(logger_level)) {
logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
logger_level = 'info';
}
logger.level = logger_level;
winston.loggers.get('console').level = logger_level;
logger.transports[2].level = logger_level;
}
function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -703,7 +700,7 @@ app.use(function(req, res, next) {
next();
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
next();
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
next();
} else {
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -742,18 +739,6 @@ const optionalJwt = async function (req, res, next) {
return next();
};
const optionalPin = async function (req, res, next) {
const use_pin = config_api.getConfigItem('ytdl_use_pin');
if (use_pin && req.path.includes('/api/setConfig')) {
if (!req.query.pin_token) {
res.sendStatus(418); // I'm a teapot (RFC 2324)
return;
}
return next();
}
return next();
};
app.get('/api/config', function(req, res) {
let config_file = config_api.getConfigFile();
res.send({
@@ -762,7 +747,7 @@ app.get('/api/config', function(req, res) {
});
});
app.post('/api/setConfig', optionalJwt, optionalPin, function(req, res) {
app.post('/api/setConfig', optionalJwt, function(req, res) {
let new_config_file = req.body.new_config_file;
if (new_config_file && new_config_file['YoutubeDLMaterial']) {
let success = config_api.setConfigFile(new_config_file);
@@ -788,7 +773,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
const db_info = await db_api.getDBStats();
res.send(db_info);
res.send({db_info: db_info});
});
app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -828,13 +813,11 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
additionalArgs: req.body.additionalArgs,
customOutput: req.body.customOutput,
selectedHeight: req.body.selectedHeight,
maxHeight: req.body.maxHeight,
customQualityConfiguration: req.body.customQualityConfiguration,
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
ui_uid: req.body.ui_uid,
cropFileSettings: req.body.cropFileSettings,
ignoreArchive: req.body.ignoreArchive
cropFileSettings: req.body.cropFileSettings
};
const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -860,7 +843,6 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
additionalArgs: req.body.additionalArgs,
customOutput: req.body.customOutput,
selectedHeight: req.body.selectedHeight,
maxHeight: req.body.maxHeight,
customQualityConfiguration: req.body.customQualityConfiguration,
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
@@ -939,15 +921,35 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
// these are returned
let files = null;
const sort = req.body.sort;
const range = req.body.range;
const text_search = req.body.text_search;
const file_type_filter = req.body.file_type_filter;
const favorite_filter = req.body.favorite_filter;
const sub_id = req.body.sub_id;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
const file_count = await db_api.getRecords('files', filter_obj, true);
files = JSON.parse(JSON.stringify(files));
res.send({
files: files,
@@ -1090,6 +1092,9 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
} else if (is_playlist) {
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
} else if (type === 'subscription') {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
} else {
// error
success = false;
@@ -1104,7 +1109,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
});
});
app.post('/api/incrementViewCount', async (req, res) => {
app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
let file_uid = req.body.file_uid;
let sub_id = req.body.sub_id;
let uuid = req.body.uuid;
@@ -1239,9 +1244,12 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
let deleteForever = req.body.deleteForever;
let file = req.body.file;
let file_uid = req.body.file_uid;
let sub = req.body.sub;
let user_uid = req.isAuthenticated() ? req.user.uid : null;
let success = await db_api.deleteFile(file_uid, deleteForever);
let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
if (success) {
res.send({
@@ -1422,9 +1430,10 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
app.post('/api/deleteFile', optionalJwt, async (req, res) => {
const uid = req.body.uid;
const blacklistMode = req.body.blacklistMode;
const uuid = req.isAuthenticated() ? req.user.uid : null;
let wasDeleted = false;
wasDeleted = await db_api.deleteFile(uid, blacklistMode);
wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
res.send(wasDeleted);
});
@@ -1456,7 +1465,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
for (let i = 0; i < files.length; i++) {
let wasDeleted = false;
wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
if (wasDeleted) {
delete_count++;
}
@@ -1517,69 +1526,20 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
});
});
app.post('/api/getArchives', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const sub_id = req.body.sub_id;
const filter_obj = {user_uid: uuid, sub_id: sub_id};
const type = req.body.type;
// we do this for file types because if type is null, that means get files of all types
if (type) filter_obj['type'] = type;
const archives = await db_api.getRecords('archives', filter_obj);
res.send({
archives: archives
});
});
app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const sub_id = req.body.sub_id;
const type = req.body.type;
let sub = req.body.sub;
let archive_dir = sub.archive;
const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
let full_archive_path = path.join(archive_dir, 'archive.txt');
if (archive_text !== null && archive_text !== undefined) {
res.setHeader('Content-type', "application/octet-stream");
res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
res.send(archive_text);
if (await fs.pathExists(full_archive_path)) {
res.sendFile(full_archive_path);
} else {
res.sendStatus(400);
res.sendStatus(404);
}
});
app.post('/api/importArchive', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const archive = req.body.archive;
const sub_id = req.body.sub_id;
const type = req.body.type;
const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
res.send({
success: !!imported_count,
imported_count: imported_count
});
});
app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const archives = req.body.archives;
let success = true;
for (const archive of archives) {
success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
}
res.send({
success: success
});
});
var upload_multer = multer({ dest: __dirname + '/appdata/' });
app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1771,8 +1731,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
app.post('/api/restartDownload', optionalJwt, async (req, res) => {
const download_uid = req.body.download_uid;
const new_download = await downloader_api.restartDownload(download_uid);
res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
const success = await downloader_api.restartDownload(download_uid);
res.send({success: success});
});
app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1849,15 +1809,6 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
res.send({success: success});
});
app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const new_options = req.body.new_options;
const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
res.send({success: success});
});
app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
const backup_dir = path.join('appdata', 'db_backup');
fs.ensureDirSync(backup_dir);
@@ -1946,23 +1897,12 @@ app.post('/api/auth/login'
, auth_api.generateJWT
, auth_api.returnAuthResponse
);
app.post('/api/auth/pinLogin'
, auth_api.passport.authenticate(['local_pin'], {})
, auth_api.generatePinJWT
, auth_api.returnPinAuthResponse
);
app.post('/api/auth/jwtAuth'
, auth_api.passport.authenticate('jwt', { session: false })
, auth_api.passport.authorize('jwt')
, auth_api.generateJWT
, auth_api.returnAuthResponse
);
app.post('/api/auth/pinAuth'
, auth_api.passport.authenticate('pin', { session: false })
, auth_api.passport.authorize('pin')
, auth_api.generatePinJWT
, auth_api.returnPinAuthResponse
);
app.post('/api/auth/changePassword', optionalJwt, async (req, res) => {
let user_uid = req.body.user_uid;
let password = req.body.new_password;
@@ -2052,100 +1992,6 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
res.send({success: success});
});
app.post('/api/setPin', function(req, res) {
const success = auth_api.setPin(req.body.new_pin);
res.send({
success: success
});
});
// notifications
app.post('/api/getNotifications', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
res.send({notifications: notifications});
});
// set notifications to read
app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
res.send({success: success});
});
app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
const uid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.removeRecord('notifications', {uid: uid});
res.send({success: success});
});
app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
res.send({success: success});
});
// rss feed
app.get('/api/rss', async function (req, res) {
if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
res.sendStatus(403);
return;
}
// these are returned
const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
const file_type_filter = req.query.file_type_filter;
const favorite_filter = req.query.favorite_filter === 'true';
const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const feed = new Feed({
title: 'Downloads',
description: 'YoutubeDL-Material downloads',
id: utils.getBaseURL(),
link: utils.getBaseURL(),
image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
generator: 'YoutubeDL-Material'
});
files.forEach(file => {
feed.addItem({
title: file.title,
link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
description: file.description,
author: [
{
name: file.uploader,
link: file.url
}
],
contributor: [],
date: file.timestamp,
// https://stackoverflow.com/a/45415677/8088021
image: file.thumbnailURL.replace('&', '&amp;')
});
});
res.send(feed.rss2());
});
// web server
app.use(function(req, res, next) {
//if the request is not html then move along
var accept = req.accepts('html', 'json', 'xml');

View File

@@ -1,91 +0,0 @@
const path = require('path');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const db_api = require('./db');
exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
const filter = {user_uid: user_uid, sub_id: sub_id};
if (type) filter['type'] = type;
const archive_items = await db_api.getRecords('archives', filter);
const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
return archive_item_lines.join('\n');
}
exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
return success;
}
exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return success;
}
exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return !!archive_item;
}
exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
let archive_import_count = 0;
const lines = archive_text.split('\n');
for (let line of lines) {
const archive_line_parts = line.trim().split(' ');
// should just be the extractor and the video ID
if (archive_line_parts.length !== 2) {
continue;
}
const extractor = archive_line_parts[0];
const id = archive_line_parts[1];
if (!extractor || !id) continue;
// we can't do a bulk write because we need to avoid duplicate archive items existing in db
const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
archive_import_count++;
}
return archive_import_count;
}
exports.importArchives = async () => {
const imported_archives = [];
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
if (!dir_to_check['archive_path']) continue;
const files_to_import = [
path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
]
for (const file_to_import of files_to_import) {
const file_exists = await fs.pathExists(file_to_import);
if (!file_exists) continue;
const archive_text = await fs.readFile(file_to_import, 'utf8');
await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
imported_archives.push(file_to_import);
}
}
return imported_archives;
}
const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
return {
extractor: extractor,
id: id,
type: type,
title: title,
user_uid: user_uid ? user_uid : null,
sub_id: sub_id ? sub_id : null,
timestamp: Date.now() / 1000,
uid: uuid()
}
}

View File

@@ -15,6 +15,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
// other required vars
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function () {
@@ -32,14 +33,7 @@ exports.initialize = function () {
saltRounds = 10;
// Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
if (!(+JWT_EXPIRATION)) {
logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
JWT_EXPIRATION = 86400;
} else {
JWT_EXPIRATION = +JWT_EXPIRATION;
}
SERVER_SECRET = null;
if (db_api.users_db.get('jwt_secret').value()) {
@@ -49,11 +43,11 @@ exports.initialize = function () {
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
const opts = {}
opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
exports.passport.use('jwt', new JwtStrategy(opts, async function(jwt_payload, done) {
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
@@ -62,21 +56,6 @@ exports.initialize = function () {
// or you could create a new account
}
}));
const pin_opts = {}
pin_opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('pin_token');
pin_opts.secretOrKey = SERVER_SECRET;
exports.passport.use('pin', new JwtStrategy(pin_opts, {
passwordField: 'pin'},
async function(username, password, done) {
if (await bcrypt.compare(password, config_api.getConfigItem('ytdl_pin_hash'))) {
return done(null, { success: true });
} else {
return done(null, false, { message: 'Incorrect pin' });
}
}
));
}
const setupRoles = async () => {
@@ -202,10 +181,6 @@ exports.login = async (username, password) => {
return await bcrypt.compare(password, user.passhash) ? user : false;
}
exports.pinLogin = async (pin) => {
return await bcrypt.compare(pin, config_api.getConfigItem('ytdl_pin_hash'));
}
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
@@ -214,14 +189,6 @@ exports.passport.use(new LocalStrategy({
}
));
exports.passport.use('local_pin', new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
async function(username, password, done) {
return done(null, await exports.pinLogin(password));
}
));
var getLDAPConfiguration = function(req, callback) {
const ldap_config = config_api.getConfigItem('ytdl_ldap_config');
const opts = {server: ldap_config};
@@ -263,14 +230,6 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.generatePinJWT = function(req, res, next) {
var payload = {
exp: Math.floor(Date.now() / 1000) + JWT_EXPIRATION
};
req.token = jwt.sign(payload, SERVER_SECRET);
next();
}
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
@@ -280,12 +239,6 @@ exports.returnAuthResponse = async function(req, res) {
});
}
exports.returnPinAuthResponse = async function(req, res) {
res.status(200).json({
pin_token: req.token
});
}
/***************************************
* Authorization: middleware that checks the
* JWT token for validity before allowing
@@ -479,13 +432,6 @@ exports.userPermissions = async function(user_uid) {
return user_permissions;
}
// pin
exports.setPin = async (new_pin) => {
const pin_hash = await bcrypt.hash(new_pin, saltRounds);
return config_api.setConfigItem('ytdl_pin_hash', pin_hash);
}
function getToken(queryParams) {
if (queryParams && queryParams.jwt) {
var parted = queryParams.jwt.split(' ');
@@ -497,7 +443,7 @@ function getToken(queryParams) {
} else {
return null;
}
}
};
function generateUserObject(userid, username, hash, auth_method = 'internal') {
let new_user = {

View File

@@ -185,6 +185,7 @@ const DEFAULT_CONFIG = {
"default_file_output": "",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true,
"max_concurrent_downloads": 5,
@@ -195,33 +196,21 @@ const DEFAULT_CONFIG = {
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"force_autoplay": false,
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true,
"enable_notifications": true,
"enable_all_notifications": true,
"allowed_notification_types": [],
"enable_rss_feed": false,
"use_pin": false,
"pin_hash": "",
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false,
"use_ntfy_API": false,
"ntfy_topic_URL": "",
"use_gotify_API": false,
"gotify_server_URL": "",
"gotify_app_token": "",
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": ""
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",

View File

@@ -30,6 +30,10 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_custom_args',
'path': 'YoutubeDLMaterial.Downloader.custom_args'
},
'ytdl_safe_download_override': {
'key': 'ytdl_safe_download_override',
'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
},
'ytdl_include_thumbnail': {
'key': 'ytdl_include_thumbnail',
'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -64,9 +68,9 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_download_only_mode',
'path': 'YoutubeDLMaterial.Extra.download_only_mode'
},
'ytdl_force_autoplay': {
'key': 'ytdl_force_autoplay',
'path': 'YoutubeDLMaterial.Extra.force_autoplay'
'ytdl_allow_autoplay': {
'key': 'ytdl_allow_autoplay',
'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
},
'ytdl_enable_downloads_manager': {
'key': 'ytdl_enable_downloads_manager',
@@ -76,30 +80,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_allow_playlist_categorization',
'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
},
'ytdl_enable_notifications': {
'key': 'ytdl_enable_notifications',
'path': 'YoutubeDLMaterial.Extra.enable_notifications'
},
'ytdl_enable_all_notifications': {
'key': 'ytdl_enable_all_notifications',
'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
},
'ytdl_allowed_notification_types': {
'key': 'ytdl_allowed_notification_types',
'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
},
'ytdl_enable_rss_feed': {
'key': 'ytdl_enable_rss_feed',
'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
},
'ytdl_use_pin': {
'key': 'ytdl_use_pin',
'path': 'YoutubeDLMaterial.Extra.use_pin'
},
'ytdl_pin_hash': {
'key': 'ytdl_pin_hash',
'path': 'YoutubeDLMaterial.Extra.pin_hash'
},
// API
'ytdl_use_api_key': {
@@ -118,6 +98,18 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_youtube_api_key',
'path': 'YoutubeDLMaterial.API.youtube_API_key'
},
'ytdl_use_twitch_api': {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_client_id': {
'key': 'ytdl_twitch_client_id',
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
},
'ytdl_twitch_client_secret': {
'key': 'ytdl_twitch_client_secret',
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -130,42 +122,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_generate_nfo_files',
'path': 'YoutubeDLMaterial.API.generate_NFO_files'
},
'ytdl_use_ntfy_API': {
'key': 'ytdl_use_ntfy_API',
'path': 'YoutubeDLMaterial.API.use_ntfy_API'
},
'ytdl_ntfy_topic_url': {
'key': 'ytdl_ntfy_topic_url',
'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
},
'ytdl_use_gotify_API': {
'key': 'ytdl_use_gotify_API',
'path': 'YoutubeDLMaterial.API.use_gotify_API'
},
'ytdl_gotify_server_url': {
'key': 'ytdl_gotify_server_url',
'path': 'YoutubeDLMaterial.API.gotify_server_URL'
},
'ytdl_gotify_app_token': {
'key': 'ytdl_gotify_app_token',
'path': 'YoutubeDLMaterial.API.gotify_app_token'
},
'ytdl_use_telegram_API': {
'key': 'ytdl_use_telegram_API',
'path': 'YoutubeDLMaterial.API.use_telegram_API'
},
'ytdl_telegram_bot_token': {
'key': 'ytdl_telegram_bot_token',
'path': 'YoutubeDLMaterial.API.telegram_bot_token'
},
'ytdl_telegram_chat_id': {
'key': 'ytdl_telegram_chat_id',
'path': 'YoutubeDLMaterial.API.telegram_chat_id'
},
'ytdl_webhook_url': {
'key': 'ytdl_webhook_url',
'path': 'YoutubeDLMaterial.API.webhook_URL'
},
// Themes
@@ -350,4 +306,4 @@ const YTDL_ARGS_WITH_VALUES = [
// we're using a Set here for performance
exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
exports.CURRENT_VERSION = 'v4.3.1';
exports.CURRENT_VERSION = 'v4.3';

View File

@@ -2,7 +2,6 @@ var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config');
var utils = require('./utils')
@@ -59,13 +58,6 @@ const tables = {
name: 'tasks',
primary_key: 'key'
},
notifications: {
name: 'notifications',
primary_key: 'uid'
},
archives: {
name: 'archives'
},
test: {
name: 'test'
}
@@ -156,7 +148,6 @@ exports._connectToDB = async (custom_connection_string = null) => {
await database.collection(table).createIndex(text_search);
}
});
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
return true;
} catch(err) {
logger.error(err);
@@ -261,16 +252,13 @@ exports.getFileDirectoriesAndDBs = async () => {
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'audio'),
user_uid: user.uid,
type: 'audio',
archive_path: utils.getArchiveFolder('audio', user.uid)
type: 'audio'
});
// add user's video dir to check list
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'video'),
user_uid: user.uid,
type: 'video',
archive_path: utils.getArchiveFolder('video', user.uid)
type: 'video'
});
}
} else {
@@ -280,15 +268,13 @@ exports.getFileDirectoriesAndDBs = async () => {
// add audio dir to check list
dirs_to_check.push({
basePath: audioFolderPath,
type: 'audio',
archive_path: utils.getArchiveFolder('audio')
type: 'audio'
});
// add video dir to check list
dirs_to_check.push({
basePath: videoFolderPath,
type: 'video',
archive_path: utils.getArchiveFolder('video')
type: 'video'
});
}
@@ -309,8 +295,7 @@ exports.getFileDirectoriesAndDBs = async () => {
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
user_uid: subscription_to_check.user_uid,
type: subscription_to_check.type,
sub_id: subscription_to_check['id'],
archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
sub_id: subscription_to_check['id']
});
}
@@ -365,7 +350,7 @@ exports.addMetadataPropertyToDB = async (property_key) => {
}
}
return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
return await exports.bulkUpdateRecords('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
@@ -462,8 +447,8 @@ exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null)
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid, uuid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
@@ -509,22 +494,16 @@ exports.deleteFile = async (uid, blacklistMode = false) => {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
// get id/extractor from JSON
const archive_path = utils.getArchiveFolder(type, uuid);
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null;
if (jsonobj) id = jsonobj.id;
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
}
if (jsonExists) await fs.unlink(jsonPath);
@@ -555,32 +534,8 @@ exports.getVideo = async (file_uid) => {
return await exports.getRecord('files', {uid: file_uid});
}
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await exports.getRecords('files', filter_obj, true);
return {files, file_count};
exports.getFiles = async (uuid = null) => {
return await exports.getRecords('files', {user_uid: uuid});
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
@@ -595,7 +550,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override
if (using_local_db) {
if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write();
return true;
}
@@ -698,15 +653,9 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
// Update
exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
exports.updateRecord = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
if (nested_mode) {
// if object is nested we need to handle it differently
update_obj = utils.convertFlatObjectToNestedObject(update_obj);
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
return true;
}
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
return true;
}
@@ -728,19 +677,7 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
return !!(output['result']['ok']);
}
exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
// local db override
if (using_local_db) {
const props_to_remove = Object.keys(remove_obj);
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
return !!(output['result']['ok']);
}
exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
// local db override
if (using_local_db) {
local_db.get(table).each((record) => {
@@ -1154,14 +1091,6 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
} else if ('$ne' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
} else if ('$lt' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
} else if ('$gt' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
} else if ('$lte' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
} else if ('$gte' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
}
} else {
// handle case of nested property check
@@ -1176,8 +1105,3 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
});
return return_val;
}
// should only be used for tests
exports.setLocalDBMode = (mode) => {
using_local_db = mode;
}

View File

@@ -1,6 +1,7 @@
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const Mutex = require('async-mutex').Mutex;
@@ -13,8 +14,6 @@ const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const notifications_api = require('./notifications');
const archive_api = require('./archive');
const mutex = new Mutex();
let should_check_downloads = true;
@@ -27,25 +26,6 @@ if (db_api.database_initialized) {
});
}
/*
This file handles all the downloading functionality.
To download a file, we go through 4 steps. Here they are with their respective index & function:
0: Create the download
- createDownload()
1: Get info for the download (we need this step for categories and archive functionality)
- collectInfo()
2: Download the file
- downloadQueuedFile()
3: Complete
- N/A
We use checkDownloads() to move downloads through the steps and call their respective functions.
*/
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
return await mutex.runExclusive(async () => {
const download = {
@@ -104,10 +84,10 @@ exports.resumeDownload = async (download_uid) => {
exports.restartDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await exports.clearDownload(download_uid);
const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
should_check_downloads = true;
return new_download;
return success;
}
exports.cancelDownload = async (download_uid) => {
@@ -126,10 +106,9 @@ exports.clearDownload = async (download_uid) => {
return await db_api.removeRecord('download_queue', {uid: download_uid});
}
async function handleDownloadError(download, error_message, error_type = null) {
if (!download || !download['uid']) return;
notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_type);
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
async function handleDownloadError(download_uid, error_message) {
if (!download_uid) return;
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
}
async function setupDownloads() {
@@ -175,13 +154,6 @@ async function checkDownloads() {
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
if (waiting_download['finished_step'] && !waiting_download['finished']) {
if (waiting_download['sub_id']) {
const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
if (sub_missing) {
handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
continue;
}
}
// move to next step
running_downloads_count++;
if (waiting_download['step_index'] === 0) {
@@ -221,20 +193,6 @@ async function collectInfo(download_uid) {
return;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
if (exists_in_archive) {
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
logger.warn(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error, 'exists_in_archive');
return;
}
}
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
@@ -245,10 +203,11 @@ async function collectInfo(download_uid) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
args = utils.filterArgs(args, ['--no-simulate']);
info = await exports.getVideoInfoByURL(url, args, download_uid);
}
const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
download['category'] = category;
// setup info required to calculate download progress
@@ -271,7 +230,6 @@ async function collectInfo(download_uid) {
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title'],
category: stripped_category,
prefetched_info: null
});
}
@@ -314,14 +272,14 @@ async function downloadQueuedFile(download_uid) {
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download, err.stderr);
await handleDownloadError(download_uid, err.stderr);
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download, error_message, 'no_output');
await handleDownloadError(download_uid, error_message);
logger.warn(error_message);
resolve(false);
return;
@@ -330,10 +288,7 @@ async function downloadQueuedFile(download_uid) {
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
// we have to do this because sometimes there will be leading characters before the actual json
const start_idx = output[i].indexOf('{"');
const clean_output = output[i].slice(start_idx, output[i].length);
output_json = JSON.parse(clean_output);
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
@@ -350,7 +305,7 @@ async function downloadQueuedFile(download_uid) {
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -386,14 +341,17 @@ async function downloadQueuedFile(download_uid) {
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj);
}
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null;
if (file_objs.length > 1) {
@@ -405,7 +363,7 @@ async function downloadQueuedFile(download_uid) {
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download, error_message, 'no_metadata');
await handleDownloadError(download_uid, error_message);
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -421,10 +379,6 @@ async function downloadQueuedFile(download_uid) {
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
}
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
@@ -472,8 +426,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
} else if (heightParam && heightParam !== '' && !is_audio) {
const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
qualityPath = ['-f', `'(mp4)[height${maxHeight ? '<' : ''}=${heightParam}]`];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
@@ -510,6 +463,28 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
@@ -552,8 +527,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
return new Promise(resolve => {
// remove bad args
const temp_args = utils.filterArgs(args, ['--no-simulate']);
const new_args = [...temp_args];
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
@@ -585,8 +559,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error, 'parse_failed');
await handleDownloadError(download_uid, error);
}
resolve(null);
}
@@ -595,8 +568,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error_message);
await handleDownloadError(download_uid, error_message);
}
resolve(null);
}
@@ -662,3 +634,13 @@ exports.generateNFOFile = (info, output_path) => {
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join('appdata', 'archives');
}
}

View File

@@ -10,7 +10,7 @@ fi
# chown current working directory to current user
if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$0" "$@"
fi

View File

@@ -1,157 +0,0 @@
const db_api = require('./db');
const config_api = require('./config');
const logger = require('./logger');
const utils = require('./utils');
const { uuid } = require('uuidv4');
const fetch = require('node-fetch');
const { gotify } = require("gotify");
const TelegramBot = require('node-telegram-bot-api');
const NOTIFICATION_TYPE_TO_TITLE = {
task_finished: 'Task finished',
download_complete: 'Download complete',
download_error: 'Download error'
}
const NOTIFICATION_TYPE_TO_BODY = {
task_finished: (notification) => notification['data']['task_title'],
download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
download_error: (notification) => {return `Error: ${notification['data']['download_error_type']}\nURL: ${notification['data']['download_url']}`}
}
const NOTIFICATION_TYPE_TO_URL = {
task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
}
const NOTIFICATION_TYPE_TO_THUMBNAIL = {
task_finished: () => null,
download_complete: (notification) => notification['data']['file_thumbnail'],
download_error: () => null
}
exports.sendNotification = async (notification) => {
// info necessary if we are using 3rd party APIs
const type = notification['type'];
const data = {
title: NOTIFICATION_TYPE_TO_TITLE[type],
body: NOTIFICATION_TYPE_TO_BODY[type](notification),
type: type,
url: NOTIFICATION_TYPE_TO_URL[type](notification),
thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
}
if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
sendNtfyNotification(data);
}
if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
sendGotifyNotification(data);
}
if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
sendTelegramNotification(data);
}
if (config_api.getConfigItem('ytdl_webhook_url')) {
sendGenericNotification(data);
}
await db_api.insertRecordIntoTable('notifications', notification);
return notification;
}
exports.sendTaskNotification = async (task_obj, confirmed) => {
if (!notificationEnabled('task_finished')) return;
// workaround for tasks which are user_uid agnostic
const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.sendDownloadNotification = async (file, user_uid) => {
if (!notificationEnabled('download_complete')) return;
const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.sendDownloadErrorNotification = async (download, user_uid, error_type = null) => {
if (!notificationEnabled('download_error')) return;
const data = {download_uid: download.uid, download_url: download.url, download_error_type: error_type};
const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.createNotification = (type, actions, data, user_uid) => {
const notification = {
type: type,
actions: actions,
data: data,
user_uid: user_uid,
uid: uuid(),
read: false,
timestamp: Date.now()/1000
}
return notification;
}
function notificationEnabled(type) {
return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
}
function sendNtfyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to ntfy');
fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
method: 'POST',
body: body,
headers: {
'Title': title,
'Tags': type,
'Click': url,
'Attach': thumbnail
}
});
}
async function sendGotifyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to gotify');
await gotify({
server: config_api.getConfigItem('ytdl_gotify_server_url'),
app: config_api.getConfigItem('ytdl_gotify_app_token'),
title: title,
message: body,
tag: type,
priority: 5, // Keeping default from docs, may want to change this,
extras: {
"client::notification": {
click: { url: url },
bigImageUrl: thumbnail
}
}
});
}
async function sendTelegramNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to Telegram');
const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
const bot = new TelegramBot(bot_token);
if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
}
function sendGenericNotification(data) {
const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
logger.verbose(`Sending generic notification to ${webhook_url}`);
fetch(webhook_url, {
method: 'POST',
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data),
});
}

2114
backend/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -27,26 +27,22 @@
"compression": "^1.7.4",
"config": "^3.2.3",
"express": "^4.17.3",
"express-session": "^1.17.3",
"feed": "^4.2.2",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"gotify": "^1.1.0",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^9.2.2",
"moment": "^2.29.4",
"moment": "^2.29.2",
"mongodb": "^3.6.9",
"multer": "1.4.5-lts.1",
"node-fetch": "^2.6.7",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"node-telegram-bot-api": "^0.61.0",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.1",
"passport-jwt": "^4.0.0",
"passport-ldapauth": "^3.0.1",
"passport-local": "^1.0.0",
"progress": "^2.0.3",

View File

@@ -3,7 +3,6 @@ const path = require('path');
const youtubedl = require('youtube-dl');
const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
@@ -139,25 +138,28 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && (await fs.pathExists(appendedBasePath))) {
if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists
// TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
if (await fs.pathExists(archive_file_path)) {
await fs.unlink(archive_file_path);
}
await fs.rmdir(sub.archive);
}
await fs.remove(appendedBasePath);
}
await db_api.removeAllRecords('archives', {sub_id: sub.id});
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
if (typeof sub === 'string') {
// TODO: fix bad workaround where sub is a sub_id
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
}
// TODO: combine this with deletefile
let basePath = null;
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
let retrievedExtractor = null;
await db_api.removeRecord('files', {uid: file_uid});
@@ -176,9 +178,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
]);
if (jsonExists) {
const info_json = fs.readJSONSync(jsonPath);
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
retrievedID = fs.readJSONSync(jsonPath)['id'];
await fs.unlink(jsonPath);
}
@@ -196,9 +196,11 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
return false;
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
if (useArchive && retrievedID) {
const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
}
return true;
}
@@ -329,6 +331,8 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = getAppendedBasePath(sub, basePath);
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -364,6 +368,21 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...customArgsArray);
}
let archive_dir = null;
let archive_path = null;
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
if (sub.type && sub.type === 'audio') {
archive_path = path.join(archive_dir, 'merged_audio.txt');
} else {
archive_path = path.join(archive_dir, 'merged_video.txt');
}
}
downloadConfig.push('--download-archive', archive_path);
}
if (sub.timerange && !redownload) {
downloadConfig.push('--dateafter', sub.timerange);
}
@@ -406,14 +425,7 @@ async function getFilesToDownload(sub, output_jsons) {
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
}
files_to_download.push(output_json);
}
}
@@ -432,12 +444,7 @@ async function getAllSubscriptions() {
}
async function getSubscription(subID) {
// stringify and parse because we may override the 'downloading' property
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
// now with the download_queue, we may need to override 'downloading'
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
return sub;
return await db_api.getRecord('subscriptions', {id: subID});
}
async function getSubscriptionByName(subName, user_uid = null) {

View File

@@ -1,7 +1,5 @@
const db_api = require('./db');
const notifications_api = require('./notifications');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const fs = require('fs-extra');
const logger = require('./logger');
@@ -35,28 +33,6 @@ const TASKS = {
confirm: youtubedl_api.updateYoutubeDL,
title: 'Update youtube-dl',
job: null
},
delete_old_files: {
run: checkForAutoDeleteFiles,
confirm: autoDeleteFiles,
title: 'Delete old files',
job: null
},
import_legacy_archives: {
run: archive_api.importArchives,
title: 'Import legacy archives',
job: null
}
}
const defaultOptions = {
all: {
auto_confirm: false
},
delete_old_files: {
blacklist_files: false,
blacklist_subscription_files: false,
threshold_days: ''
}
}
@@ -69,7 +45,7 @@ function scheduleJob(task_key, schedule) {
const dayOfWeek = schedule['data']['dayOfWeek'] != null ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] != null ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] != null ? schedule['data']['minute'] : null;
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
} else {
logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
return null;
@@ -81,7 +57,7 @@ function scheduleJob(task_key, schedule) {
logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
return;
}
// remove schedule if it's a one-time task
if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
// we're just "running" the task, any confirmation should be user-initiated
@@ -101,10 +77,9 @@ exports.setupTasks = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
const task_in_db = await db_api.getRecord('tasks', {key: task_key});
if (!task_in_db) {
// insert task metadata into table if missing, eventually move title to UI
// insert task metadata into table if missing
await db_api.insertRecordIntoTable('tasks', {
key: task_key,
title: TASKS[task_key]['title'],
@@ -115,19 +90,9 @@ exports.setupTasks = async () => {
data: null,
error: null,
schedule: null,
options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
options: {}
});
} else {
// verify all options exist in task
for (const key of Object.keys(mergedDefaultOptions)) {
const option_key = `options.${key}`;
// Remove any potential mangled option keys (#861)
await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
}
}
// reset task if necessary
await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
@@ -158,23 +123,15 @@ exports.executeTask = async (task_key) => {
exports.executeRun = async (task_key) => {
logger.verbose(`Running task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {error: null})
// don't set running to true when backup up DB as it will be stick "running" if restored
if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
const data = await TASKS[task_key].run();
await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
logger.verbose(`Finished running task ${task_key}`);
const task_obj = await db_api.getRecord('tasks', {key: task_key});
await notifications_api.sendTaskNotification(task_obj, false);
if (task_obj['options'] && task_obj['options']['auto_confirm']) {
exports.executeConfirm(task_key);
}
}
exports.executeConfirm = async (task_key) => {
logger.verbose(`Confirming task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {error: null})
if (!TASKS[task_key]['confirm']) {
return null;
}
@@ -184,7 +141,6 @@ exports.executeConfirm = async (task_key) => {
await TASKS[task_key].confirm(data);
await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
logger.verbose(`Finished confirming task ${task_key}`);
await notifications_api.sendTaskNotification(task_obj, false);
}
exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -237,31 +193,4 @@ async function removeDuplicates(data) {
}
}
// auto delete files
async function checkForAutoDeleteFiles() {
const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
const error_message = 'Failed to do delete check because no limit was set!';
logger.error(error_message);
await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
return null;
}
const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
return {files_to_remove: files_to_remove};
}
async function autoDeleteFiles(data) {
const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
if (data['files_to_remove']) {
logger.info(`Removing ${data['files_to_remove'].length} old files!`);
for (let i = 0; i < data['files_to_remove'].length; i++) {
const file_to_remove = data['files_to_remove'][i];
await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
}
}
}
exports.TASKS = TASKS;

View File

@@ -3,6 +3,7 @@ const low = require('lowdb')
const winston = require('winston');
const path = require('path');
process.chdir('./backend')
const FileSync = require('lowdb/adapters/FileSync');
@@ -37,7 +38,6 @@ var auth_api = require('../authentication/auth');
var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const archive_api = require('../archive');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
@@ -66,12 +66,12 @@ const sample_video_json = {
describe('Database', async function() {
describe('Import', async function() {
// it('Migrate', async function() {
// await db_api.connectToDB();
// await db_api.removeAllRecords();
// const success = await db_api.importJSONToDB(db.value(), users_db.value());
// assert(success);
// });
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
@@ -104,199 +104,157 @@ describe('Database', async function() {
});
});
describe('Export', function() {
});
describe('Basic functions', async function() {
// test both local_db and remote_db
const local_db_modes = [false, true];
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
for (const local_db_mode of local_db_modes) {
let use_local_db = local_db_mode;
describe(`Use local DB - ${use_local_db}`, async function() {
beforeEach(async function() {
if (!use_local_db) {
this.timeout(120000);
await db_api.connectToDB(0);
}
await db_api.removeAllRecords('test');
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Add and read record - Nested property', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
assert(added_record['test_add'] === 'test');
assert(!not_added_record);
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Replace filter', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
assert(count === 1);
await db_api.removeRecord('test', {test_replace_filter: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
it('Remove property from record', async function() {
await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
assert(updated_record['test_keep']);
assert(!updated_record['test_remove']);
await db_api.removeRecord('test', {test_keep: 'test'});
});
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Remove records', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
assert(delete_succeeded);
const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
assert(count === 0);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
console.log(random_record)
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
console.log(`Query time: ${(query_end - query_start)/1000}s`);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = !!random_record;
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
}
assert(success);
});
});
describe('Local DB Filters', async function() {
@@ -331,6 +289,8 @@ describe('Multi User', async function() {
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
});
describe('Authentication', function() {
@@ -508,7 +468,7 @@ describe('Downloader', function() {
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1710641401';
const example_vod = '1493770675';
it('Download VOD', async function() {
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
@@ -633,72 +593,30 @@ describe('Tasks', function() {
});
describe('Archive', async function() {
const archive_path = path.join('test', 'archives');
fs.ensureDirSync(archive_path);
const archive_file_path = path.join(archive_path, 'archive_video.txt');
const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
fs.writeFileSync(blacklist_file_path, '');
});
it('Delete from archive', async function() {
await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
const new_archive = fs.readFileSync(archive_file_path);
assert(!new_archive.includes('testing2'));
});
afterEach(async function() {
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
});
it('Import archive', async function() {
const archive_text = `
testextractor1 testing1
testextractor1 testing2
testextractor2 testing1
testextractor1 testing3
`;
const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
assert(count === 4)
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
console.log(archive_items);
assert(archive_items.length === 4);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
assert(success);
});
it('Get archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(archive_item1 && archive_item2);
});
it('Archive duplicates', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
assert(count === 3);
});
it('Remove from archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
assert(success);
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
assert(!!archive_item1);
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(!archive_item2);
const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
assert(!!archive_item3);
it('Delete from archive - blacklist', async function() {
await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
const new_archive = fs.readFileSync(archive_file_path);
const new_blacklist = fs.readFileSync(blacklist_file_path);
assert(!new_archive.includes('testing2'));
assert(new_blacklist.includes('testing2'));
});
});
@@ -708,24 +626,4 @@ describe('Utils', async function() {
const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
});
it('Convert flat object to nested object', async function() {
// No modfication
const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
// Standard setup
const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
// Nested branches
const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
});
});

View File

@@ -4,28 +4,19 @@ const logger = require('./logger');
const moment = require('moment');
const fs = require('fs-extra')
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
async function getCommentsForVOD(vodId) {
async function getCommentsForVOD(clientID, clientSecret, vodId) {
const { promisify } = require('util');
const child_process = require('child_process');
const exec = promisify(child_process.exec);
// Reject invalid params to prevent command injection attack
if (!vodId.match(/^[0-9a-z]+$/)) {
logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
return null;
}
const is_windows = process.platform === 'win32';
const cliExt = is_windows ? '.exe' : ''
const cliPath = `TwitchDownloaderCLI${cliExt}`
if (!fs.existsSync(cliPath)) {
logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
return null;
}
const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -82,7 +73,9 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const chat = await getCommentsForVOD(vodId);
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
// save file if needed params are included
let file_path = null;

View File

@@ -4,7 +4,6 @@ const ffmpeg = require('fluent-ffmpeg');
const archiver = require('archiver');
const fetch = require('node-fetch');
const ProgressBar = require('progress');
const winston = require('winston');
const config_api = require('./config');
const logger = require('./logger');
@@ -13,7 +12,7 @@ const CONSTS = require('./consts');
const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
exports.getTrueFileName = (unfixed_path, type) => {
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -28,13 +27,13 @@ exports.getTrueFileName = (unfixed_path, type) => {
return fixed_path;
}
exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
// return empty array if the path doesn't exist
if (!(await fs.pathExists(basePath))) return [];
let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = await exports.recFindByExt(basePath, ext);
var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) {
let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -42,33 +41,33 @@ exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false)
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = await exports.getJSONByType(type, id, basePath);
var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue;
if (full_metadata) {
jsonobj['id'] = id;
files.push(jsonobj);
continue;
}
var upload_date = exports.formatDateString(jsonobj.upload_date);
var upload_date = formatDateString(jsonobj.upload_date);
var isaudio = type === 'audio';
var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
files.push(file_obj);
}
return files;
}
exports.createContainerZipFile = async (file_name, container_file_objs) => {
async function createContainerZipFile(file_name, container_file_objs) {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
}
exports.createZipFile = async (zip_file_path, file_paths) => {
async function createZipFile(zip_file_path, file_paths) {
let output = fs.createWriteStream(zip_file_path);
var archive = archiver('zip', {
@@ -96,7 +95,7 @@ exports.createZipFile = async (zip_file_path, file_paths) => {
return zip_file_path;
}
exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
function getJSONMp4(name, customPath, openReadPerms = false) {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
var jsonPath = path.join(customPath, name + ".info.json");
@@ -111,7 +110,7 @@ exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
return obj;
}
exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
function getJSONMp3(name, customPath, openReadPerms = false) {
var obj = null;
if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
var jsonPath = path.join(customPath, name + ".info.json");
@@ -128,11 +127,11 @@ exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
return obj;
}
exports.getJSON = (file_path, type) => {
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -143,12 +142,12 @@ exports.getJSON = (file_path, type) => {
return obj;
}
exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
exports.getDownloadedThumbnail = (file_path) => {
const file_path_no_extension = exports.removeFileExtension(file_path);
function getDownloadedThumbnail(file_path) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
@@ -164,7 +163,7 @@ exports.getDownloadedThumbnail = (file_path) => {
return null;
}
exports.getExpectedFileSize = (input_info_jsons) => {
function getExpectedFileSize(input_info_jsons) {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
@@ -187,12 +186,12 @@ exports.getExpectedFileSize = (input_info_jsons) => {
return expected_filesize;
}
exports.fixVideoMetadataPerms = (file_path, type) => {
function fixVideoMetadataPerms(file_path, type) {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = exports.removeFileExtension(file_path);
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
@@ -209,10 +208,10 @@ exports.fixVideoMetadataPerms = (file_path, type) => {
}
}
exports.deleteJSONFile = (file_path, type) => {
function deleteJSONFile(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = exports.removeFileExtension(file_path);
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
@@ -221,7 +220,58 @@ exports.deleteJSONFile = (file_path, type) => {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
exports.durationStringToNumber = (dur_str) => {
// archive helper functions
async function removeIDFromArchive(archive_path, type, id) {
const archive_file = path.join(archive_path, `archive_${type}.txt`);
const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
if (lastIndex === -1) return null;
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_file, updatedData);
if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
}
async function writeToBlacklist(archive_folder, type, line) {
let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
// adds newline to the beginning of the line
line.replace('\n', '');
line.replace('\r', '');
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
const archive_file = path.join(archive_path, `archive_${type}.txt`);
if (await fs.pathExists(archive_path)) {
const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
} else {
logger.info(`Could not find archive file for file ${uid}. Creating...`);
await fs.close(await fs.open(archive_file, 'w'));
}
}
function durationStringToNumber(dur_str) {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
@@ -231,22 +281,23 @@ exports.durationStringToNumber = (dur_str) => {
return num_sum;
}
exports.getMatchingCategoryFiles = (category, files) => {
function getMatchingCategoryFiles(category, files) {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
exports.addUIDsToCategory = (category, files) => {
const files_that_match = exports.getMatchingCategoryFiles(category, files);
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
exports.getCurrentDownloader = () => {
function getCurrentDownloader() {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
return details_json['downloader'];
}
exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
async function recFindByExt(base, ext, files, result, recursive = true)
{
files = files || (await fs.readdir(base))
result = result || []
@@ -255,7 +306,7 @@ exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
if ( (await fs.stat(newbase)).isDirectory() )
{
if (!recursive) continue;
result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
}
else
{
@@ -268,17 +319,17 @@ exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
return result
}
exports.removeFileExtension = (filename) => {
function removeFileExtension(filename) {
const filename_parts = filename.split('.');
filename_parts.splice(filename_parts.length - 1);
return filename_parts.join('.');
}
exports.formatDateString = (date_string) => {
function formatDateString(date_string) {
return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
}
exports.createEdgeNGrams = (str) => {
function createEdgeNGrams(str) {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
@@ -300,7 +351,7 @@ exports.createEdgeNGrams = (str) => {
// ffmpeg helper functions
exports.cropFile = async (file_path, start, end, ext) => {
async function cropFile(file_path, start, end, ext) {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
let base_ffmpeg_call = ffmpeg(file_path);
@@ -329,13 +380,13 @@ exports.cropFile = async (file_path, start, end, ext) => {
* setTimeout, but its a promise.
* @param {number} ms
*/
exports.wait = async (ms) => {
async function wait(ms) {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
exports.checkExistsWithTimeout = async (filePath, timeout) => {
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
@@ -364,7 +415,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
}
// helper function to download file using fetch
exports.fetchFile = async (url, path, file_label) => {
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
@@ -391,7 +442,7 @@ exports.fetchFile = async (url, path, file_label) => {
});
}
exports.restartServer = async (is_update = false) => {
async function restartServer(is_update = false) {
logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
// the following line restarts the server through pm2
@@ -404,7 +455,7 @@ exports.restartServer = async (is_update = false) => {
// - if already exists and doesn't have value, ignore
// - if it doesn't exist and has value, add both arg and value
// - if it doesn't exist and doesn't have value, add arg
exports.injectArgs = (original_args, new_args) => {
function injectArgs(original_args, new_args) {
const updated_args = original_args.slice();
try {
for (let i = 0; i < new_args.length; i++) {
@@ -432,11 +483,11 @@ exports.injectArgs = (original_args, new_args) => {
return updated_args;
}
exports.filterArgs = (args, args_to_remove) => {
function filterArgs(args, args_to_remove) {
return args.filter(x => !args_to_remove.includes(x));
}
exports.searchObjectByString = (o, s) => {
const searchObjectByString = function(o, s) {
s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
s = s.replace(/^\./, ''); // strip a leading dot
var a = s.split('.');
@@ -451,7 +502,7 @@ exports.searchObjectByString = (o, s) => {
return o;
}
exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
function stripPropertiesFromObject(obj, properties, whitelist = false) {
if (!whitelist) {
const new_obj = JSON.parse(JSON.stringify(obj));
for (let field of properties) {
@@ -467,7 +518,7 @@ exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
return new_obj;
}
exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
function getArchiveFolder(type, user_uid = null, sub = null) {
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
const subsFolderPath = config_api.getConfigItem('ytdl_subscriptions_base_path');
@@ -486,38 +537,6 @@ exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
}
}
exports.getBaseURL = () => {
return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
}
exports.updateLoggerLevel = (new_logger_level) => {
const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
if (!possible_levels.includes(new_logger_level)) {
logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
new_logger_level = 'info';
}
logger.level = new_logger_level;
winston.loggers.get('console').level = new_logger_level;
logger.transports[2].level = new_logger_level;
}
exports.convertFlatObjectToNestedObject = (obj) => {
const result = {};
for (const key in obj) {
const nestedKeys = key.split('.');
let currentObj = result;
for (let i = 0; i < nestedKeys.length; i++) {
if (i === nestedKeys.length - 1) {
currentObj[nestedKeys[i]] = obj[key];
} else {
currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
currentObj = currentObj[nestedKeys[i]];
}
}
}
return result;
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -535,7 +554,38 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
this.view_count = view_count;
this.height = height;
this.abr = abr;
this.favorite = false;
}
exports.File = File;
}
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive: removeIDFromArchive,
writeToBlacklist: writeToBlacklist,
deleteFileFromArchive: deleteFileFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
checkExistsWithTimeout: checkExistsWithTimeout,
fetchFile: fetchFile,
restartServer: restartServer,
injectArgs: injectArgs,
filterArgs: filterArgs,
searchObjectByString: searchObjectByString,
stripPropertiesFromObject: stripPropertiesFromObject,
getArchiveFolder: getArchiveFolder,
File: File
}

View File

@@ -21,4 +21,4 @@ version: 0.1.0
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "4.3.1"
appVersion: "4.3"

File diff suppressed because one or more lines are too long

View File

@@ -18,7 +18,7 @@ services:
- "8998:17442"
image: tzahi12345/youtubedl-material:latest
ytdl-mongo-db:
image: mongo:4
image: mongo
logging:
driver: "none"
container_name: mongo-db

View File

@@ -1,53 +0,0 @@
import platform
import requests
import shutil
import os
import re
from github import Github
machine = platform.machine()
def isARM():
return True if machine.startswith('arm') else False
def getLatestFileInRepo(repo, search_string):
# Create an unauthenticated instance of the Github object
g = Github(os.environ.get('GH_TOKEN'))
# Replace with the repository owner and name
repo = g.get_repo(repo)
# Get all releases of the repository
releases = repo.get_releases()
# Loop through the releases in reverse order (from latest to oldest)
for release in list(releases):
# Get the release assets (files attached to the release)
assets = release.get_assets()
# Loop through the assets
for asset in assets:
if re.search(search_string, asset.name):
print(f'Downloading: {asset.name}')
response = requests.get(asset.browser_download_url)
with open(asset.name, 'wb') as f:
f.write(response.content)
print(f'Download complete: {asset.name}. Unzipping...')
shutil.unpack_archive(asset.name, './')
print(f'Unzipping complete!')
os.remove(asset.name)
break
else:
continue
break
else:
# If no matching release is found, print a message
print(f'No release found with {search_string}')
def getLatestCLIRelease():
isArm = isARM()
searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
getLatestFileInRepo("lay295/TwitchDownloader", searchString)
getLatestCLIRelease()

View File

@@ -26,7 +26,7 @@ apt-get update && apt-get -y install curl xz-utils
echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
curl -o ffmpeg.txz \
--connect-timeout 5 \
--max-time 120 \
--max-time 10 \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \

5684
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "youtube-dl-material",
"version": "4.3.1",
"version": "4.3.0",
"license": "MIT",
"scripts": {
"ng": "ng",
@@ -21,21 +21,21 @@
},
"private": true,
"dependencies": {
"@angular-devkit/core": "^15.0.1",
"@angular/animations": "^15.0.1",
"@angular/cdk": "^15.0.0",
"@angular/common": "^15.0.1",
"@angular/compiler": "^15.0.1",
"@angular/core": "^15.0.1",
"@angular/forms": "^15.0.1",
"@angular/localize": "^15.0.1",
"@angular/material": "^15.0.0",
"@angular/platform-browser": "^15.0.1",
"@angular/platform-browser-dynamic": "^15.0.1",
"@angular/router": "^15.0.1",
"@angular-devkit/core": "^14.0.4",
"@angular/animations": "^14.0.4",
"@angular/cdk": "^14.0.4",
"@angular/common": "^14.0.4",
"@angular/compiler": "^14.0.4",
"@angular/core": "^14.0.4",
"@angular/forms": "^14.0.4",
"@angular/localize": "^14.0.4",
"@angular/material": "^14.0.4",
"@angular/platform-browser": "^14.0.4",
"@angular/platform-browser-dynamic": "^14.0.4",
"@angular/router": "^14.0.4",
"@fontsource/material-icons": "^4.5.4",
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^6.0.0",
"@videogular/ngx-videogular": "^5.0.1",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
@@ -44,29 +44,29 @@
"fs-extra": "^10.0.0",
"material-icons": "^1.10.8",
"nan": "^2.14.1",
"ngx-avatars": "^1.4.1",
"ng-lazyload-image": "^7.0.1",
"ngx-avatars": "^1.3.1",
"ngx-file-drop": "^13.0.0",
"rxjs": "^6.6.3",
"rxjs-compat": "^6.0.0-rc.0",
"tslib": "^2.0.0",
"typescript": "~4.8.4",
"typescript": "~4.6.3",
"xliff-to-json": "^1.0.4",
"zone.js": "~0.11.4"
},
"devDependencies": {
"@angular-devkit/build-angular": "^15.0.1",
"@angular/cli": "^15.0.1",
"@angular/compiler-cli": "^15.0.1",
"@angular/language-service": "^15.0.1",
"@angular-devkit/build-angular": "^14.0.4",
"@angular/cli": "^14.0.4",
"@angular/compiler-cli": "^14.0.4",
"@angular/language-service": "^14.0.4",
"@types/core-js": "^2.5.2",
"@types/file-saver": "^2.0.1",
"@types/jasmine": "~3.6.0",
"@types/node": "^12.11.1",
"@typescript-eslint/eslint-plugin": "^4.29.0",
"@typescript-eslint/parser": "^4.29.0",
"ajv": "^7.2.4",
"codelyzer": "^6.0.0",
"electron": "^19.1.9",
"electron": "^19.0.6",
"eslint": "^7.32.0",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
@@ -76,7 +76,7 @@
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"openapi-typescript-codegen": "^0.23.0",
"openapi-typescript-codegen": "^0.21.0",
"protractor": "~7.0.0",
"ts-node": "~3.0.4",
"tslint": "~6.1.0"

View File

@@ -3,7 +3,6 @@
/* eslint-disable */
export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest';
export type { Archive } from './models/Archive';
export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest';
export type { binary } from './models/binary';
export type { body_19 } from './models/body_19';
@@ -27,10 +26,8 @@ export type { DatabaseFile } from './models/DatabaseFile';
export { DBBackup } from './models/DBBackup';
export type { DBInfoResponse } from './models/DBInfoResponse';
export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse';
export type { DeleteArchiveItemsRequest } from './models/DeleteArchiveItemsRequest';
export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest';
export type { DeletePlaylistRequest } from './models/DeletePlaylistRequest';
export type { DeleteSubscriptionFileRequest } from './models/DeleteSubscriptionFileRequest';
export type { DeleteUserRequest } from './models/DeleteUserRequest';
@@ -53,8 +50,6 @@ export type { GetAllFilesRequest } from './models/GetAllFilesRequest';
export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
export type { GetArchivesRequest } from './models/GetArchivesRequest';
export type { GetArchivesResponse } from './models/GetArchivesResponse';
export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
export type { GetDownloadRequest } from './models/GetDownloadRequest';
export type { GetDownloadResponse } from './models/GetDownloadResponse';
@@ -68,7 +63,6 @@ export type { GetLogsRequest } from './models/GetLogsRequest';
export type { GetLogsResponse } from './models/GetLogsResponse';
export type { GetMp3sResponse } from './models/GetMp3sResponse';
export type { GetMp4sResponse } from './models/GetMp4sResponse';
export type { GetNotificationsResponse } from './models/GetNotificationsResponse';
export type { GetPlaylistRequest } from './models/GetPlaylistRequest';
export type { GetPlaylistResponse } from './models/GetPlaylistResponse';
export type { GetPlaylistsRequest } from './models/GetPlaylistsRequest';
@@ -79,24 +73,16 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
export type { GetTaskRequest } from './models/GetTaskRequest';
export type { GetTaskResponse } from './models/GetTaskResponse';
export type { GetUsersResponse } from './models/GetUsersResponse';
export type { ImportArchiveRequest } from './models/ImportArchiveRequest';
export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
export type { inline_response_200_15 } from './models/inline_response_200_15';
export type { LoginRequest } from './models/LoginRequest';
export type { LoginResponse } from './models/LoginResponse';
export type { Notification } from './models/Notification';
export { NotificationAction } from './models/NotificationAction';
export { NotificationType } from './models/NotificationType';
export type { PinLoginResponse } from './models/PinLoginResponse';
export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse';
export type { RestartDownloadResponse } from './models/RestartDownloadResponse';
export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
export { Schedule } from './models/Schedule';
export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
export type { SetPinRequest } from './models/SetPinRequest';
export type { SharingToggle } from './models/SharingToggle';
export type { Sort } from './models/Sort';
export type { SubscribeRequest } from './models/SubscribeRequest';
@@ -122,10 +108,8 @@ export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
export type { UpdaterStatus } from './models/UpdaterStatus';
export type { UpdateServerRequest } from './models/UpdateServerRequest';
export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest';
export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
export type { UpdateUserRequest } from './models/UpdateUserRequest';
export type { UploadCookiesRequest } from './models/UploadCookiesRequest';
export type { User } from './models/User';
export { UserPermission } from './models/UserPermission';
export type { Version } from './models/Version';

View File

@@ -5,4 +5,4 @@
export type AddFileToPlaylistRequest = {
file_uid: string;
playlist_id: string;
};
};

View File

@@ -1,16 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { FileType } from './FileType';
export type Archive = {
extractor: string;
id: string;
type: FileType;
title: string;
user_uid?: string;
sub_id?: string;
timestamp: number;
uid: string;
};

View File

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
export type BaseChangePermissionsRequest = {
permission: UserPermission;
new_value: YesNo;
};
};

View File

@@ -12,4 +12,4 @@ export type Category = {
* Overrides file output for downloaded files in category
*/
custom_output?: string;
};
};

View File

@@ -22,4 +22,4 @@ export namespace CategoryRule {
}
}
}

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
role: string;
});
});

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
user_uid: string;
});
});

View File

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
* UID of the concurrent stream
*/
uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
export type CheckConcurrentStreamResponse = {
stream: ConcurrentStream;
};
};

View File

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
clear_finished?: boolean;
clear_paused?: boolean;
clear_errors?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type Config = {
YoutubeDLMaterial: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { Config } from './Config';
export type ConfigResponse = {
config_file: Config;
success: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type CreateCategoryRequest = {
name: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Category } from './Category';
export type CreateCategoryResponse = {
new_category?: Category;
success?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
playlistName: string;
uids: Array<string>;
thumbnailURL: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
export type CreatePlaylistResponse = {
new_playlist: Playlist;
success: boolean;
};
};

View File

@@ -5,4 +5,4 @@
export type CropFileSettings = {
cropFileStart: number;
cropFileEnd: number;
};
};

View File

@@ -17,4 +17,4 @@ export namespace DBBackup {
}
}
}

View File

@@ -14,6 +14,5 @@ subscriptions?: TableInfo;
users?: TableInfo;
roles?: TableInfo;
download_queue?: TableInfo;
archives?: TableInfo;
};
};
};

View File

@@ -26,7 +26,6 @@ export type DatabaseFile = {
path: string;
upload_date: string;
uid: string;
user_uid?: string;
sharingEnabled?: boolean;
category?: Category;
view_count?: number;
@@ -41,5 +40,4 @@ export type DatabaseFile = {
* In Kbps
*/
abr?: number;
favorite: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
* Number of files removed
*/
delete_count?: number;
};
};

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { Archive } from './Archive';
export type DeleteArchiveItemsRequest = {
archives: Array<Archive>;
};

View File

@@ -4,4 +4,4 @@
export type DeleteCategoryRequest = {
category_uid: string;
};
};

View File

@@ -5,4 +5,4 @@
export type DeleteMp3Mp4Request = {
uid: string;
blacklistMode?: boolean;
};
};

View File

@@ -1,7 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type DeleteNotificationRequest = {
uid: string;
};

View File

@@ -4,4 +4,4 @@
export type DeletePlaylistRequest = {
playlist_id: string;
};
};

View File

@@ -2,10 +2,14 @@
/* tslint:disable */
/* eslint-disable */
import type { SubscriptionRequestData } from './SubscriptionRequestData';
export type DeleteSubscriptionFileRequest = {
file_uid: string;
file: string;
file_uid?: string;
sub: SubscriptionRequestData;
/**
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteUserRequest = {
uid: string;
};
};

View File

@@ -19,12 +19,8 @@ export type Download = {
* Error text, set if download fails.
*/
error?: string | null;
/**
* Error type, may or may not be set in case of an error
*/
error_type?: string | null;
user_uid?: string;
sub_id?: string;
sub_name?: string;
prefetched_info?: any;
};
};

View File

@@ -2,9 +2,8 @@
/* tslint:disable */
/* eslint-disable */
import type { FileType } from './FileType';
export type DownloadArchiveRequest = {
type?: FileType;
sub_id?: string;
sub: {
archive_dir: string;
};
};

View File

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
playlist_id?: string;
url?: string;
type?: FileType;
};
};

View File

@@ -35,18 +35,10 @@ export type DownloadRequest = {
* Height of the video, if known
*/
selectedHeight?: string;
/**
* Max height that should be used, useful for playlists. selectedHeight will override this.
*/
maxHeight?: string;
/**
* Specify ffmpeg/avconv audio quality
*/
maxBitrate?: string;
type?: FileType;
cropFileSettings?: CropFileSettings;
/**
* If using youtube-dl archive, download will ignore it
*/
ignoreArchive?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type DownloadResponse = {
download?: Download;
};
};

View File

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
export type DownloadTwitchChatByVODIDResponse = {
chat: Array<TwitchChatMessage>;
};
};

View File

@@ -4,4 +4,4 @@
export type DownloadVideosForSubscriptionRequest = {
subID: string;
};
};

View File

@@ -5,4 +5,4 @@
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}
}

View File

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
AUDIO_ONLY = 'audio_only',
VIDEO_ONLY = 'video_only',
BOTH = 'both',
}
}

View File

@@ -4,4 +4,4 @@
export type GenerateArgsResponse = {
args?: Array<string>;
};
};

View File

@@ -4,4 +4,4 @@
export type GenerateNewApiKeyResponse = {
new_api_key: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Category } from './Category';
export type GetAllCategoriesResponse = {
categories: Array<Category>;
};
};

View File

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
* Filters downloads with the array
*/
uids?: Array<string> | null;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetAllDownloadsResponse = {
downloads?: Array<Download>;
};
};

View File

@@ -13,12 +13,8 @@ export type GetAllFilesRequest = {
*/
text_search?: string;
file_type_filter?: FileTypeFilter;
/**
* If set to true, only gets favorites
*/
favorite_filter?: boolean;
/**
* Include if you want to filter by subscription
*/
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
export type GetAllSubscriptionsResponse = {
subscriptions: Array<Subscription>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Task } from './Task';
export type GetAllTasksResponse = {
tasks?: Array<Task>;
};
};

View File

@@ -1,10 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { FileType } from './FileType';
export type GetArchivesRequest = {
type?: FileType;
sub_id?: string;
};

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { Archive } from './Archive';
export type GetArchivesResponse = {
archives: Array<Archive>;
};

View File

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
export type GetDBBackupsResponse = {
tasks?: Array<DBBackup>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetDownloadRequest = {
download_uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetDownloadResponse = {
download?: Download;
};
};

View File

@@ -4,4 +4,4 @@
export type GetFileFormatsRequest = {
url?: string;
};
};

View File

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
result: {
formats?: Array<any>;
};
};
};

View File

@@ -14,4 +14,4 @@ export type GetFileRequest = {
* User UID
*/
uuid?: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { DatabaseFile } from './DatabaseFile';
export type GetFileResponse = {
success: boolean;
file?: DatabaseFile;
};
};

View File

@@ -16,4 +16,4 @@ export type GetFullTwitchChatRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -5,4 +5,4 @@
export type GetFullTwitchChatResponse = {
success: boolean;
error?: string;
};
};

View File

@@ -4,4 +4,4 @@
export type GetLogsRequest = {
lines?: number;
};
};

View File

@@ -8,4 +8,4 @@ export type GetLogsResponse = {
*/
logs?: string;
success?: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type GetMp3sResponse = {
* All audio playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -11,4 +11,4 @@ export type GetMp4sResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -1,9 +0,0 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { Notification } from './Notification';
export type GetNotificationsResponse = {
notifications?: Array<Notification>;
};

View File

@@ -9,4 +9,4 @@ export type GetPlaylistRequest = {
type?: FileType;
uuid?: string;
include_file_metadata?: boolean;
};
};

View File

@@ -12,4 +12,4 @@ export type GetPlaylistResponse = {
* File objects for every uid in the playlist's uids property, in the same order
*/
file_objs?: Array<DatabaseFile>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetPlaylistsRequest = {
include_categories?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Playlist } from './Playlist';
export type GetPlaylistsResponse = {
playlists: Array<Playlist>;
};
};

Some files were not shown because too many files have changed in this diff Show More