Compare commits

..

13 Commits

Author SHA1 Message Date
Tzahi12345
0a608d8b7a uncaught exceptions now reported as errors by electron-logger 2023-04-20 00:19:28 -04:00
Tzahi12345
8e1ebb5a4a Prevent electron from crashing on error 2023-04-20 00:18:41 -04:00
Tzahi12345
bf38d9e67e Frontend initialization refactor to ensure app start in electron 2023-04-20 00:12:39 -04:00
Tzahi12345
a7c041aae1 More electron error logging 2023-04-19 02:40:11 -04:00
Tzahi12345
a2ae9db1c6 Added additional electron logging 2023-04-19 02:35:24 -04:00
Tzahi12345
4aa98916ed Electron app opens if server is already open
Added logging to electron
2023-04-19 02:30:17 -04:00
Tzahi12345
2098cc542c Ensure process is killed on electron close 2023-04-19 02:07:18 -04:00
Tzahi12345
26de55fe86 Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into desktop-app 2023-04-17 23:57:13 -04:00
Tzahi12345
08c5647521 Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into desktop-app 2023-04-17 23:55:06 -04:00
Tzahi12345
a7507aa803 Added icon for mac 2023-04-17 23:53:43 -04:00
Tzahi12345
0c2937695c Added 512px logo 2023-04-17 23:18:07 -04:00
Tzahi12345
4fd676d50c Added splash screen 2023-04-17 22:51:33 -04:00
Tzahi12345
771fe3d985 Moved electron to backend 2023-04-17 00:48:30 -04:00
214 changed files with 4431 additions and 4601 deletions

View File

@@ -15,9 +15,9 @@ jobs:
- name: checkout code
uses: actions/checkout@v2
- name: setup node
uses: actions/setup-node@v3
uses: actions/setup-node@v2
with:
node-version: '16'
node-version: '14'
cache: 'npm'
- name: install dependencies
run: |
@@ -33,7 +33,7 @@ jobs:
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -55,7 +55,7 @@ jobs:
Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
- name: upload build artifact
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v1
with:
name: youtubedl-material
path: build

View File

@@ -18,21 +18,10 @@ jobs:
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v1
- name: build & push images
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
#platforms: linux/amd64
push: false
tags: tzahi12345/youtubedl-material:nightly-pr
- name: Build docker images
run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

View File

@@ -27,7 +27,7 @@ jobs:
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -60,10 +60,10 @@ jobs:
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64/v8
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}

View File

@@ -34,7 +34,7 @@ jobs:
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -44,7 +44,7 @@ jobs:
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Generate Docker image metadata
id: docker-meta
@@ -63,7 +63,7 @@ jobs:
type=sha,prefix=sha-,format=short
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}

1
.gitignore vendored
View File

@@ -67,6 +67,7 @@ backend/appdata/users.json
backend/users/*
backend/appdata/cookies.txt
backend/public
backend/dist
src/assets/i18n/*.json
# User Files

14
.vscode/launch.json vendored
View File

@@ -4,20 +4,6 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Dev: Debug Backend",
"request": "launch",
"runtimeArgs": [
"run-script",
"debug"
],
"runtimeExecutable": "npm",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"cwd": "${workspaceFolder}/backend"
},
{
"type": "node",
"request": "attach",

View File

@@ -1,38 +0,0 @@
<h1>Development</h1>
- [First time...](#first-time)
- [Setup](#setup)
- [Startup](#startup)
- [Debugging the backend (VSC)](#debugging-the-backend-vsc)
- [Deploy changes](#deploy-changes)
- [Frontend](#frontend)
- [Backend](#backend)
# First time...
## Setup
Checkout the repository and navigate to the `youtubedl-material` directory.
```bash
vim ./src/assets/default.json # Edit settings for your local environment. This config file is just the dev config file, if YTDL_MODE is not set to "debug", then ./backend/appdata/default.json will be used
npm -g install pm2 # Install pm2
npm install # Install dependencies for the frontend
cd ./backend
npm install # Install dependencies for the backend
cd ..
npm run build # Build the frontend
```
This step have to be done only once.
## Startup
Navigate to the `youtubedl-material/backend` directory and run `npm start`.
# Debugging the backend (VSC)
Open the `youtubedl-material` directory in Visual Studio Code and run the launch configuration `Dev: Debug Backend`.
# Deploy changes
## Frontend
Navigate to the `youtubedl-material` directory and run `npm run build`. Restart the backend.
## Backend
Simply restart the backend.

View File

@@ -1,17 +1,15 @@
# Fetching our utils
FROM ubuntu:22.04 AS utils
# Fetching our ffmpeg
FROM ubuntu:22.04 AS ffmpeg
ENV DEBIAN_FRONTEND=noninteractive
# Use script due local build compability
COPY docker-utils/*.sh .
RUN chmod +x *.sh
COPY ffmpeg-fetch.sh .
RUN chmod +x ffmpeg-fetch.sh
RUN sh ./ffmpeg-fetch.sh
RUN sh ./fetch-twitchdownloader.sh
# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
# Go to 20.04
FROM ubuntu:22.04 AS base
ARG TARGETPLATFORM
FROM ubuntu:20.04 AS base
ARG DEBIAN_FRONTEND=noninteractive
ENV UID=1000
ENV GID=1000
@@ -19,30 +17,19 @@ ENV USER=youtube
ENV NO_UPDATE_NOTIFIER=true
ENV PM2_HOME=/app/pm2
ENV ALLOW_CONFIG_MUTATIONS=true
# Directy fetch specific version
## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb
RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
apt update && \
apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 && \
apt install -y --no-install-recommends curl ca-certificates tzdata && \
curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
apt install -y --no-install-recommends nodejs && \
npm -g install npm n && \
n 16.14.2 && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN case ${TARGETPLATFORM} in \
"linux/amd64") NODE_ARCH=amd64 ;; \
"linux/arm") NODE_ARCH=armhf ;; \
"linux/arm/v7") NODE_ARCH=armhf ;; \
"linux/arm64") NODE_ARCH=arm64 ;; \
esac \
&& curl -L https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_$NODE_ARCH.deb -o ./nodejs.deb && \
apt update && \
apt install -y ./nodejs.deb && \
apt clean && \
rm -rf /var/lib/apt/lists/* &&\
rm nodejs.deb;
# Build frontend
ARG BUILDPLATFORM
FROM --platform=${BUILDPLATFORM} node:16 as frontend
FROM base as frontend
RUN npm install -g @angular/cli
WORKDIR /build
COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
@@ -50,8 +37,6 @@ COPY [ "src/", "/build/src/" ]
RUN npm install && \
npm run build && \
ls -al /build/backend/public
RUN npm uninstall -g @angular/cli
RUN rm -rf node_modules
# Install backend deps
@@ -62,35 +47,22 @@ RUN npm config set strict-ssl false && \
npm install --prod && \
ls -al
#FROM base as python
# armv7 need build from source
#WORKDIR /app
#COPY docker-utils/GetTwitchDownloader.py .
#RUN apt update && \
# apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip python3-dev build-essential libffi-dev && \
# apt clean && \
# rm -rf /var/lib/apt/lists/*
#RUN pip install PyGithub requests
#RUN python GetTwitchDownloader.py
# Final image
FROM base
RUN npm install -g pm2 && \
apt update && \
apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
pip install pycryptodomex && \
apt remove -y --purge build-essential && \
apt autoremove -y --purge && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install tdh-tcd pycryptodomex
WORKDIR /app
# User 1000 already exist from base image
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/usr/local/bin/TwitchDownloaderCLI"]
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
#COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"]
RUN chown $UID:$GID .
RUN chmod +x /app/fix-scripts/*.sh
# Add some persistence data
#VOLUME ["/app/appdata"]

View File

@@ -1758,14 +1758,14 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
required:
- task_key
UpdateTaskScheduleRequest:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_schedule:
$ref: '#/components/schemas/Schedule'
required:
@@ -1775,7 +1775,7 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_data:
type: object
required:
@@ -1785,7 +1785,7 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_options:
type: object
required:
@@ -2726,7 +2726,7 @@ components:
type: object
properties:
key:
$ref: '#/components/schemas/TaskType'
type: string
title:
type: string
last_ran:
@@ -2742,20 +2742,9 @@ components:
error:
type: string
schedule:
$ref: '#/components/schemas/Schedule'
type: object
options:
type: object
TaskType:
type: string
enum:
- backup_local_db
- missing_files_check
- missing_db_records
- duplicate_files_check
- youtubedl_update_check
- delete_old_files
- import_legacy_archives
- rebuild_database
Schedule:
required:
- type
@@ -2888,7 +2877,6 @@ components:
- sharing
- advanced_download
- downloads_manager
- tasks_manager
YesNo:
type: string
enum:

View File

@@ -28,28 +28,13 @@ Dark mode:
NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
Required dependencies:
* Node.js 16
* Python
Optional dependencies:
* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
* [Twitch Downloader CLI](https://github.com/lay295/TwitchDownloader) (for downloading Twitch VOD chats)
<details>
<summary>Debian/Ubuntu</summary>
Debian/Ubuntu:
```bash
curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
```
</details>
<details>
<summary>CentOS 7</summary>
CentOS 7:
```bash
sudo yum install epel-release
@@ -57,11 +42,13 @@ sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfu
sudo yum install centos-release-scl-rh
sudo yum install rh-nodejs12
scl enable rh-nodejs12 bash
curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash -
sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
```
</details>
Optional dependencies:
* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
### Installing
@@ -85,9 +72,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm run build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
Lastly, type `npm -g install pm2` to install pm2 globally.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.

View File

@@ -1,7 +1,12 @@
// TODO: ignore this if not in electron
const rootPath = require('electron-root-path').rootPath;
process.chdir(rootPath);
const { uuid } = require('uuidv4');
const fs = require('fs-extra');
const { promisify } = require('util');
const auth_api = require('./authentication/auth');
const winston = require('winston');
const path = require('path');
const compression = require('compression');
const multer = require('multer');
@@ -18,7 +23,6 @@ const CONSTS = require('./consts')
const read_last_lines = require('read-last-lines');
const ps = require('ps-node');
const Feed = require('feed').Feed;
const session = require('express-session');
// needed if bin/details somehow gets deleted
if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -34,7 +38,6 @@ const categories_api = require('./categories');
const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
var app = express();
@@ -57,48 +60,7 @@ let debugMode = process.env.YTDL_MODE === 'debug';
const admin_token = '4241b401-7236-493e-92b5-b72696b9d853';
// logging setup
config_api.initialize();
db_api.initialize(db, users_db);
auth_api.initialize(db_api);
// Set some defaults
db.defaults(
{
playlists: [],
files: [],
configWriteFlag: false,
downloads: {},
subscriptions: [],
files_to_db_migration_complete: false,
tasks_manager_role_migration_complete: false,
archives_migration_complete: false
}).write();
users_db.defaults(
{
users: [],
roles: {
"admin": {
"permissions": [
'filemanager',
'settings',
'subscriptions',
'sharing',
'advanced_download',
'downloads_manager'
]
}, "user": {
"permissions": [
'filemanager',
'subscriptions',
'sharing'
]
}
}
}
).write();
// required initialization
// config values
let url = null;
@@ -150,21 +112,61 @@ if (fs.existsSync('version.json')) {
version_info = {'type': 'N/A', 'tag': 'N/A', 'commit': 'N/A', 'date': 'N/A'};
}
// don't overwrite config if it already happened.. NOT
// let alreadyWritten = db.get('configWriteFlag').value();
exports.initialize = () => {
config_api.initialize();
db_api.initialize(db, users_db);
auth_api.initialize(db_api);
// checks if config exists, if not, a config is auto generated
config_api.configExistsCheck();
// Set some defaults
db.defaults(
{
playlists: [],
files: [],
configWriteFlag: false,
downloads: {},
subscriptions: [],
files_to_db_migration_complete: false,
tasks_manager_role_migration_complete: false,
archives_migration_complete: false
}).write();
setAndLoadConfig();
users_db.defaults(
{
users: [],
roles: {
"admin": {
"permissions": [
'filemanager',
'settings',
'subscriptions',
'sharing',
'advanced_download',
'downloads_manager'
]
}, "user": {
"permissions": [
'filemanager',
'subscriptions',
'sharing'
]
}
}
}
).write();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
// checks if config exists, if not, a config is auto generated
config_api.configExistsCheck();
// use passport
app.use(auth_api.passport.initialize());
app.use(session({ secret: uuid(), resave: true, saveUninitialized: true }))
app.use(auth_api.passport.session());
setAndLoadConfig();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
// use passport
app.use(auth_api.passport.initialize());
}
exports.initialize();
// actual functions
@@ -175,10 +177,10 @@ async function checkMigrations() {
if (!simplified_db_migration_complete) {
logger.info('Beginning migration: 4.1->4.2+')
let success = await simplifyDBFileStructure();
success = success && await files_api.addMetadataPropertyToDB('view_count');
success = success && await files_api.addMetadataPropertyToDB('description');
success = success && await files_api.addMetadataPropertyToDB('height');
success = success && await files_api.addMetadataPropertyToDB('abr');
success = success && await db_api.addMetadataPropertyToDB('view_count');
success = success && await db_api.addMetadataPropertyToDB('description');
success = success && await db_api.addMetadataPropertyToDB('height');
success = success && await db_api.addMetadataPropertyToDB('abr');
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
if (success) { logger.info('4.1->4.2+ migration complete!'); }
@@ -254,7 +256,7 @@ async function simplifyDBFileStructure() {
return true;
}
async function startServer() {
exports.startServer = async () => {
if (process.env.USING_HEROKU && process.env.PORT) {
// default to heroku port if using heroku
backendPort = process.env.PORT || backendPort;
@@ -535,7 +537,6 @@ async function loadConfig() {
if (allowSubscriptions) {
// set downloading to false
let subscriptions = await subscriptions_api.getAllSubscriptions();
subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
// runs initially, then runs every ${subscriptionCheckInterval} seconds
const watchSubscriptionsInterval = function() {
@@ -548,7 +549,9 @@ async function loadConfig() {
}
// start the server here
startServer();
if (typeof require !== 'undefined' && require.main === module) {
exports.startServer();
}
return true;
}
@@ -727,7 +730,7 @@ const optionalJwt = async function (req, res, next) {
const uuid = using_body ? req.body.uuid : req.query.uuid;
const uid = using_body ? req.body.uid : req.query.uid;
const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
if (file) {
req.can_watch = true;
return next();
@@ -938,7 +941,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
const sub_id = req.body.sub_id;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
res.send({
files: files,
@@ -1104,7 +1107,7 @@ app.post('/api/incrementViewCount', async (req, res) => {
uuid = req.user.uid;
}
const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
const file_obj = await db_api.getVideo(file_uid, uuid, sub_id);
const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
const new_view_count = current_view_count + 1;
@@ -1232,7 +1235,7 @@ app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
let deleteForever = req.body.deleteForever;
let file_uid = req.body.file_uid;
let success = await files_api.deleteFile(file_uid, deleteForever);
let success = await db_api.deleteFile(file_uid, deleteForever);
if (success) {
res.send({
@@ -1320,7 +1323,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
let playlistName = req.body.playlistName;
let uids = req.body.uids;
const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
res.send({
new_playlist: new_playlist,
@@ -1333,13 +1336,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
let include_file_metadata = req.body.include_file_metadata;
const playlist = await files_api.getPlaylist(playlist_id, uuid);
const playlist = await db_api.getPlaylist(playlist_id, uuid);
const file_objs = [];
if (playlist && include_file_metadata) {
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await files_api.getVideo(uid, uuid);
const file_obj = await db_api.getVideo(uid, uuid);
if (file_obj) file_objs.push(file_obj);
// TODO: remove file from playlist if could not be found
}
@@ -1377,7 +1380,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
playlist.uids.push(file_uid);
let success = await files_api.updatePlaylist(playlist);
let success = await db_api.updatePlaylist(playlist);
res.send({
success: success
});
@@ -1385,7 +1388,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
let playlist = req.body.playlist;
let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
res.send({
success: success
});
@@ -1415,7 +1418,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => {
const blacklistMode = req.body.blacklistMode;
let wasDeleted = false;
wasDeleted = await files_api.deleteFile(uid, blacklistMode);
wasDeleted = await db_api.deleteFile(uid, blacklistMode);
res.send(wasDeleted);
});
@@ -1447,7 +1450,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
for (let i = 0; i < files.length; i++) {
let wasDeleted = false;
wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode);
wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
if (wasDeleted) {
delete_count++;
}
@@ -1473,10 +1476,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
if (playlist_id) {
zip_file_generated = true;
const playlist_files_to_download = [];
const playlist = await files_api.getPlaylist(playlist_id, uuid);
const playlist = await db_api.getPlaylist(playlist_id, uuid);
for (let i = 0; i < playlist['uids'].length; i++) {
const playlist_file_uid = playlist['uids'][i];
const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
playlist_files_to_download.push(file_obj);
}
@@ -1490,7 +1493,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
// generate zip
file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
} else {
const file_obj = await files_api.getVideo(uid, uuid, sub_id)
const file_obj = await db_api.getVideo(uid, uuid, sub_id)
file_path_to_download = file_obj.path;
}
if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
@@ -1637,12 +1640,14 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
file_obj = await files_api.getVideo(uid, uuid, sub_id);
file_obj = await db_api.getVideo(uid, uuid, sub_id);
if (file_obj) file_path = file_obj['path'];
else file_path = null;
}
if (!fs.existsSync(file_path)) {
logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj.id}`);
res.sendStatus(404);
return;
}
const stat = fs.statSync(file_path);
const fileSize = stat.size;
@@ -1929,38 +1934,9 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
// user authentication
app.post('/api/auth/register', optionalJwt, async (req, res) => {
const userid = req.body.userid;
const username = req.body.username;
const plaintextPassword = req.body.password;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
res.sendStatus(409);
return;
}
if (plaintextPassword === "") {
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
res.sendStatus(409);
return;
}
if (!userid || !username) {
logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
}
const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
if (!new_user) {
res.sendStatus(409);
return;
}
res.send({
user: new_user
});
});
app.post('/api/auth/register'
, optionalJwt
, auth_api.registerUser);
app.post('/api/auth/login'
, auth_api.passport.authenticate(['local', 'ldapauth'], {})
, auth_api.generateJWT
@@ -2012,7 +1988,18 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
app.post('/api/deleteUser', optionalJwt, async (req, res) => {
let uid = req.body.uid;
try {
const success = await auth_api.deleteUser(uid);
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
}
res.send({success: success});
} catch (err) {
logger.error(err);
@@ -2103,7 +2090,7 @@ app.get('/api/rss', async function (req, res) {
const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const feed = new Feed({
title: 'Downloads',

View File

@@ -23,12 +23,7 @@
"download_only_mode": false,
"allow_autoplay": true,
"enable_downloads_manager": true,
"allow_playlist_categorization": true,
"force_autoplay": false,
"enable_notifications": true,
"enable_all_notifications": true,
"allowed_notification_types": [],
"enable_rss_feed": false
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
@@ -40,18 +35,7 @@
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false,
"use_ntfy_API": false,
"ntfy_topic_URL": "",
"use_gotify_API": false,
"gotify_server_URL": "",
"gotify_app_token": "",
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": ""
"generate_NFO_files": false
},
"Themes": {
"default_theme": "default",

View File

@@ -1,13 +1,11 @@
const config_api = require('../config');
const CONSTS = require('../consts');
const consts = require('../consts');
const logger = require('../logger');
const db_api = require('../db');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
const fs = require('fs-extra');
const path = require('path');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -18,7 +16,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = 10;
let saltRounds = null;
exports.initialize = function () {
/*************************
@@ -33,6 +31,8 @@ exports.initialize = function () {
});
}
saltRounds = 10;
// Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
if (!(+JWT_EXPIRATION)) {
@@ -68,7 +68,14 @@ exports.initialize = function () {
const setupRoles = async () => {
const required_roles = {
admin: {
permissions: CONSTS.AVAILABLE_PERMISSIONS
permissions: [
'filemanager',
'settings',
'subscriptions',
'sharing',
'advanced_download',
'downloads_manager'
]
},
user: {
permissions: [
@@ -106,41 +113,55 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
exports.registerUser = async (userid, username, plaintextPassword) => {
const hash = await bcrypt.hash(plaintextPassword, saltRounds);
const new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
return null;
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
return null;
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
return new_user;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
res.sendStatus(409);
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
return;
}
}
exports.deleteUser = async (uid) => {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
if (plaintextPassword === "") {
res.sendStatus(400);
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
return;
}
return success;
bcrypt.hash(plaintextPassword, saltRounds)
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
});
}
})
.then(function(result) {
})
.catch(function(err) {
logger.error(err);
if( err.code == 'ER_DUP_ENTRY' ) {
res.status(409).send('UserId already taken');
} else {
res.sendStatus(409);
}
});
}
/***************************************
@@ -221,7 +242,7 @@ exports.returnAuthResponse = async function(req, res) {
user: req.user,
token: req.token,
permissions: await exports.userPermissions(req.user.uid),
available_permissions: CONSTS.AVAILABLE_PERMISSIONS
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -305,7 +326,7 @@ exports.getUserVideos = async function(user_uid, type) {
}
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {uid: file_uid});
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -392,8 +413,8 @@ exports.userPermissions = async function(user_uid) {
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

View File

@@ -208,6 +208,9 @@ const DEFAULT_CONFIG = {
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false,
@@ -219,9 +222,7 @@ const DEFAULT_CONFIG = {
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": "",
"webhook_URL": ""
},
"Themes": {
"default_theme": "default",

View File

@@ -110,6 +110,18 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_youtube_api_key',
'path': 'YoutubeDLMaterial.API.youtube_API_key'
},
'ytdl_use_twitch_api': {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_client_id': {
'key': 'ytdl_twitch_client_id',
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
},
'ytdl_twitch_client_secret': {
'key': 'ytdl_twitch_client_secret',
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -158,14 +170,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_webhook_url',
'path': 'YoutubeDLMaterial.API.webhook_URL'
},
'ytdl_discord_webhook_url': {
'key': 'ytdl_discord_webhook_url',
'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
},
'ytdl_slack_webhook_url': {
'key': 'ytdl_slack_webhook_url',
'path': 'YoutubeDLMaterial.API.slack_webhook_URL'
},
// Themes
@@ -347,11 +351,7 @@ const YTDL_ARGS_WITH_VALUES = [
'--convert-subs'
];
exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
// we're using a Set here for performance
exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
exports.CURRENT_VERSION = 'v4.3.1';

View File

@@ -1,11 +1,11 @@
const fs = require('fs-extra')
const path = require('path')
var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config');
const utils = require('./utils')
var utils = require('./utils')
const logger = require('./logger');
const low = require('lowdb')
@@ -167,9 +167,82 @@ exports._connectToDB = async (custom_connection_string = null) => {
}
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
// TODO: check if video exists, throw error if not
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
return file_object;
}
function generateFileObject(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
} else if (!jsonobj['_filename']) {
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
var stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = utils.formatDateString(jsonobj.upload_date);
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function getAppendedBasePathSub(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
exports.getFileDirectoriesAndDBs = async () => {
@@ -244,6 +317,277 @@ exports.getFileDirectoriesAndDBs = async () => {
return dirs_to_check;
}
exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it
const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) {
// add additional info
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
}
}
}
return imported_files;
}
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
const first_video = await exports.getVideo(uids[0]);
const thumbnailToUse = first_video['thumbnailURL'];
let new_playlist = {
name: playlist_name,
uids: uids,
id: uuid(),
thumbnailURL: thumbnailToUse,
registered: Date.now(),
randomize_order: false
};
new_playlist.user_uid = user_uid ? user_uid : undefined;
await exports.insertRecordIntoTable('playlists', new_playlist);
const duration = await exports.calculatePlaylistDuration(new_playlist);
await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = await exports.getRecord('playlists', {id: playlist_id});
if (!playlist) {
playlist = await exports.getRecord('categories', {uid: playlist_id});
if (playlist) {
const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
playlist['uids'] = uids;
playlist['auto'] = true;
}
}
// converts playlists to new UID-based schema
if (playlist && playlist['fileNames'] && !playlist['uids']) {
playlist['uids'] = [];
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i];
const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
}
exports.updatePlaylist(playlist, user_uid);
}
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
return playlist;
}
exports.updatePlaylist = async (playlist) => {
let playlistID = playlist.id;
const duration = await exports.calculatePlaylistDuration(playlist);
playlist.duration = duration;
return await exports.updateRecord('playlists', {id: playlistID}, playlist);
}
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
}
if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
}
exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
if (!playlist_file_objs) {
playlist_file_objs = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await exports.getVideo(uid);
if (file_obj) playlist_file_objs.push(file_obj);
}
}
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[uid]) {
try {
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
config_api.descriptors[uid][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
// get id/extractor from JSON
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
await exports.removeRecord('files', {uid: uid});
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = async (file_id, uuid = null) => {
const file_obj = await exports.getRecord('files', {id: file_id});
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid) => {
return await exports.getRecord('files', {uid: file_uid});
}
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await exports.getRecords('files', filter_obj, true);
return {files, file_count};
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
// TODO: check if video exists, throw error if not
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
}
// Basic DB functions
// Create
@@ -376,14 +720,7 @@ exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false
exports.updateRecords = async (table, filter_obj, update_obj) => {
// local db override
if (using_local_db) {
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').each((record) => {
const props_to_update = Object.keys(update_obj);
for (let i = 0; i < props_to_update.length; i++) {
const prop_to_update = props_to_update[i];
const prop_value = update_obj[prop_to_update];
record[prop_to_update] = prop_value;
}
}).write();
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').assign(update_obj).write();
return true;
}

View File

@@ -13,7 +13,6 @@ const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const files_api = require('./files');
const notifications_api = require('./notifications');
const archive_api = require('./archive');
@@ -129,7 +128,7 @@ exports.clearDownload = async (download_uid) => {
async function handleDownloadError(download, error_message, error_type = null) {
if (!download || !download['uid']) return;
notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_type);
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
}
@@ -222,7 +221,6 @@ async function collectInfo(download_uid) {
return;
}
// in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
@@ -247,10 +245,11 @@ async function collectInfo(download_uid) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
args = utils.filterArgs(args, ['--no-simulate']);
info = await exports.getVideoInfoByURL(url, args, download_uid);
}
const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
download['category'] = category;
// setup info required to calculate download progress
@@ -273,7 +272,6 @@ async function collectInfo(download_uid) {
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title'],
category: stripped_category,
prefetched_info: null
});
}
@@ -316,7 +314,7 @@ async function downloadQueuedFile(download_uid) {
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download, err.stderr, 'unknown_error');
await handleDownloadError(download, err.stderr);
resolve(false);
return;
} else if (output) {
@@ -352,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -386,9 +384,10 @@ async function downloadQueuedFile(download_uid) {
}
// registers file in DB
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
@@ -400,7 +399,7 @@ async function downloadQueuedFile(download_uid) {
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
@@ -553,8 +552,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
return new Promise(resolve => {
// remove bad args
const temp_args = utils.filterArgs(args, ['--no-simulate']);
const new_args = [...temp_args];
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
@@ -597,7 +595,7 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
logger.error(error_message);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error_message, 'info_retrieve_failed');
await handleDownloadError(download, error_message);
}
resolve(null);
}

View File

@@ -1,7 +1,17 @@
#!/bin/bash
#!/bin/sh
set -eu
CMD="npm start"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then
set -- "$CMD" "$@"
fi
# chown current working directory to current user
echo "[entrypoint] setup permission, this may take a while"
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$@"
if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$0" "$@"
fi
exec "$@"

View File

@@ -1,350 +0,0 @@
const fs = require('fs-extra')
const path = require('path')
const { uuid } = require('uuidv4');
const config_api = require('./config');
const db_api = require('./db');
const archive_api = require('./archive');
const utils = require('./utils')
const logger = require('./logger');
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
const path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']})
return file_object;
}
function generateFileObject(file_path, type) {
const jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
} else if (!jsonobj['_filename']) {
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
return null;
}
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
const stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
const title = jsonobj.title;
const url = jsonobj.webpage_url;
const uploader = jsonobj.uploader;
const upload_date = utils.formatDateString(jsonobj.upload_date);
const size = stats.size;
const thumbnail = jsonobj.thumbnail;
const duration = jsonobj.duration;
const isaudio = type === 'audio';
const description = jsonobj.description;
const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it
const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) {
// add additional info
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
}
}
}
return imported_files;
}
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
const first_video = await exports.getVideo(uids[0]);
const thumbnailToUse = first_video['thumbnailURL'];
let new_playlist = {
name: playlist_name,
uids: uids,
id: uuid(),
thumbnailURL: thumbnailToUse,
registered: Date.now(),
randomize_order: false
};
new_playlist.user_uid = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('playlists', new_playlist);
const duration = await exports.calculatePlaylistDuration(new_playlist);
await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = await db_api.getRecord('playlists', {id: playlist_id});
if (!playlist) {
playlist = await db_api.getRecord('categories', {uid: playlist_id});
if (playlist) {
const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
playlist['uids'] = uids;
playlist['auto'] = true;
}
}
// converts playlists to new UID-based schema
if (playlist && playlist['fileNames'] && !playlist['uids']) {
playlist['uids'] = [];
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i];
const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
}
exports.updatePlaylist(playlist, user_uid);
}
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
return playlist;
}
exports.updatePlaylist = async (playlist) => {
let playlistID = playlist.id;
const duration = await exports.calculatePlaylistDuration(playlist);
playlist.duration = duration;
return await db_api.updateRecord('playlists', {id: playlistID}, playlist);
}
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj);
}
if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
}
exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
if (!playlist_file_objs) {
playlist_file_objs = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await exports.getVideo(uid);
if (file_obj) playlist_file_objs.push(file_obj);
}
}
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[uid]) {
try {
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
config_api.descriptors[uid][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive || file_obj.sub_id) {
// get id/extractor from JSON
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
if (!blacklistMode) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id)
} else {
const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
if (!exists_in_archive) {
await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id);
}
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
await db_api.removeRecord('files', {uid: uid});
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = async (file_id, uuid = null) => {
const file_obj = await db_api.getRecord('files', {id: file_id});
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid) => {
return await db_api.getRecord('files', {uid: file_uid});
}
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await db_api.getRecords('files', filter_obj, true);
return {files, file_count};
}

82
backend/main.js Normal file
View File

@@ -0,0 +1,82 @@
const { app, BrowserWindow } = require('electron');
const path = require('path');
const elogger = require('electron-log');
const server = require('./app');
let win;
let splashWindow;
async function createSplashWindow() {
splashWindow = new BrowserWindow({
width: 800,
height: 600,
frame: false,
alwaysOnTop: true,
webPreferences: {
nodeIntegration: true
}
})
await splashWindow.loadFile('public/assets/splash.html')
splashWindow.on('closed', () => {
splashWindow = null
})
}
function createMainWindow() {
win = new BrowserWindow(
{
width: 800,
height: 600,
webPreferences: {
nodeIntegration: true,
contextIsolation: true,
preload: path.join(__dirname, 'preload.js')
},
icon: path.join(__dirname, 'favicon.ico'),
show: false
});
// The following is optional and will open the DevTools:
// win.webContents.openDevTools()
win.on('closed', () => {
win = null;
});
}
function loadPage() {
splashWindow.close()
// load the dist folder from Angular
win.loadURL('http://localhost:17442')
win.show()
}
async function createWindow() {
await createSplashWindow();
elogger.info('Spawning server.')
// serverProcess = spawn('node', [path.join(__dirname, 'app.js')]);
await server.startServer();
elogger.info('Done spawning!')
createMainWindow();
loadPage();
}
app.on('ready', createWindow);
// on macOS, closing the window doesn't quit the app
app.on('window-all-closed', () => {
if (process.platform !== 'darwin') {
app.quit();
}
});
// initialize the app's main window
app.on('activate', () => {
if (win === null) {
createWindow();
}
});
process.on('uncaughtException', (error) => {
elogger.error(error.message);
});

View File

@@ -2,16 +2,12 @@ const db_api = require('./db');
const config_api = require('./config');
const logger = require('./logger');
const utils = require('./utils');
const consts = require('./consts');
const { uuid } = require('uuidv4');
const fetch = require('node-fetch');
const { gotify } = require("gotify");
const TelegramBot = require('node-telegram-bot-api');
const REST = require('@discordjs/rest').REST;
const API = require('@discordjs/core').API;
const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
const NOTIFICATION_TYPE_TO_TITLE = {
task_finished: 'Task finished',
@@ -22,7 +18,7 @@ const NOTIFICATION_TYPE_TO_TITLE = {
const NOTIFICATION_TYPE_TO_BODY = {
task_finished: (notification) => notification['data']['task_title'],
download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
download_error: (notification) => {return `Error: ${notification['data']['download_error_message']}\nError code: ${notification['data']['download_error_type']}\n\nOriginal URL: ${notification['data']['download_url']}`}
download_error: (notification) => {return `Error: ${notification['data']['download_error_type']}\nURL: ${notification['data']['download_url']}`}
}
const NOTIFICATION_TYPE_TO_URL = {
@@ -61,12 +57,6 @@ exports.sendNotification = async (notification) => {
if (config_api.getConfigItem('ytdl_webhook_url')) {
sendGenericNotification(data);
}
if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
sendDiscordNotification(data);
}
if (config_api.getConfigItem('ytdl_slack_webhook_url')) {
sendSlackNotification(data);
}
await db_api.insertRecordIntoTable('notifications', notification);
return notification;
@@ -89,9 +79,9 @@ exports.sendDownloadNotification = async (file, user_uid) => {
return await exports.sendNotification(notification);
}
exports.sendDownloadErrorNotification = async (download, user_uid, error_message, error_type = null) => {
exports.sendDownloadErrorNotification = async (download, user_uid, error_type = null) => {
if (!notificationEnabled('download_error')) return;
const data = {download_uid: download.uid, download_url: download.url, download_error_message: error_message, download_error_type: error_type};
const data = {download_uid: download.uid, download_url: download.url, download_error_type: error_type};
const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
return await exports.sendNotification(notification);
}
@@ -154,88 +144,6 @@ async function sendTelegramNotification({body, title, type, url, thumbnail}) {
bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
}
async function sendDiscordNotification({body, title, type, url, thumbnail}) {
const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
const url_split = discord_webhook_url.split('webhooks/');
const [webhook_id, webhook_token] = url_split[1].split('/');
const rest = new REST({ version: '10' });
const api = new API(rest);
const embed = new EmbedBuilder()
.setTitle(title)
.setColor(0x00FFFF)
.setURL(url)
.setDescription(`ID: ${type}`);
if (thumbnail) embed.setThumbnail(thumbnail);
if (type === 'download_error') embed.setColor(0xFC2003);
const result = await api.webhooks.execute(webhook_id, webhook_token, {
content: body,
username: 'YoutubeDL-Material',
avatar_url: consts.ICON_URL,
embeds: [embed],
});
return result;
}
function sendSlackNotification({body, title, type, url, thumbnail}) {
const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
const data = {
blocks: [
{
type: "section",
text: {
type: "mrkdwn",
text: `*${title}*`
}
},
{
type: "section",
text: {
type: "plain_text",
text: body
}
}
]
}
// add thumbnail if exists
if (thumbnail) {
data['blocks'].push({
type: "image",
image_url: thumbnail,
alt_text: "notification_thumbnail"
});
}
data['blocks'].push(
{
type: "section",
text: {
type: "mrkdwn",
text: `<${url}|${url}>`
}
},
{
type: "context",
elements: [
{
type: "mrkdwn",
text: `*ID:* ${type}`
}
]
}
);
fetch(slack_webhook_url, {
method: 'POST',
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data),
});
}
function sendGenericNotification(data) {
const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
logger.verbose(`Sending generic notification to ${webhook_url}`);

2733
backend/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -2,11 +2,44 @@
"name": "backend",
"version": "1.0.0",
"description": "backend for YoutubeDL-Material",
"main": "index.js",
"main": "main.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "pm2-runtime --raw pm2.config.js",
"debug": "set YTDL_MODE=debug && node app.js"
"debug": "set YTDL_MODE=debug && node app.js",
"electron": "electron main.js",
"pack": "electron-builder --dir",
"build": "electron-builder"
},
"build": {
"appId": "youtubedl.material",
"productName": "YoutubeDL-Material GUI",
"directories": {
"output": "dist"
},
"mac": {
"category": "public.app-category.utilities",
"icon": "../src/assets/images/logo_512px.png",
"target": "dmg"
},
"win": {
"target": "nsis",
"icon": "../src/favicon.ico"
},
"files": [
"!audio/*",
"!video/*",
"!users/*",
"!subscriptions/*",
"!appdata/*",
"*.js",
"authentication/auth.js",
"main.js",
"public/**/*",
"ffmpeg*",
"ffprobe*"
],
"asar": false
},
"repository": {
"type": "git",
@@ -17,29 +50,24 @@
"bugs": {
"url": ""
},
"engines": {
"node": "^16",
"npm": "6.14.4"
},
"homepage": "",
"dependencies": {
"@discordjs/builders": "^1.6.1",
"@discordjs/core": "^0.5.2",
"app-root-path": "^3.1.0",
"archiver": "^5.3.1",
"async": "^3.2.3",
"async-mutex": "^0.4.0",
"async-mutex": "^0.3.1",
"axios": "^0.21.2",
"bcryptjs": "^2.4.0",
"command-exists": "^1.2.9",
"compression": "^1.7.4",
"config": "^3.2.3",
"express": "^4.18.2",
"express-session": "^1.17.3",
"electron-log": "^4.4.8",
"electron-root-path": "^1.1.0",
"express": "^4.17.3",
"feed": "^4.2.2",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"gotify": "^1.1.0",
"jsonwebtoken": "^9.0.0",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
@@ -48,10 +76,10 @@
"mongodb": "^3.6.9",
"multer": "1.4.5-lts.1",
"node-fetch": "^2.6.7",
"node-id3": "^0.2.6",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"node-telegram-bot-api": "^0.61.0",
"passport": "^0.6.0",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.1",
"passport-ldapauth": "^3.0.1",
@@ -62,9 +90,14 @@
"rxjs": "^7.3.0",
"shortid": "^2.2.15",
"unzipper": "^0.10.10",
"uuidv4": "^6.2.13",
"uuidv4": "^6.0.6",
"winston": "^3.7.2",
"xmlbuilder2": "^3.0.2",
"youtube-dl": "^3.0.2"
},
"devDependencies": {
"electron": "^24.1.2",
"electron-builder": "^23.6.0",
"electron-packager": "^17.1.1"
}
}

5
backend/preload.js Normal file
View File

@@ -0,0 +1,5 @@
const { contextBridge } = require('electron');
const path = require('path');
// Expose the 'path' module to the renderer process
contextBridge.exposeInMainWorld('path', path);

View File

@@ -6,21 +6,20 @@ const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const debugMode = process.env.YTDL_MODE === 'debug';
const db_api = require('./db');
const downloader_api = require('./downloader');
exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
async function subscribe(sub, user_uid = null) {
const result_obj = {
success: false,
error: ''
};
return new Promise(async resolve => {
// sub should just have url and name. here we will get isPlaylist and path
sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -35,11 +34,10 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = skip_get_info ? true : await getSubscriptionInfo(sub);
exports.writeSubscriptionMetadata(sub);
let success = await getSubscriptionInfo(sub);
if (success) {
if (!sub.paused) exports.getVideosForSub(sub, user_uid);
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
}
@@ -94,10 +92,7 @@ async function getSubscriptionInfo(sub) {
}
// if it's now valid, update
if (sub.name) {
let sub_name = sub.name;
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
if (sub_name_exists) sub_name += ` - ${sub.id}`;
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
}
}
@@ -111,7 +106,7 @@ async function getSubscriptionInfo(sub) {
});
}
exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
async function unsubscribe(sub, deleteMode, user_uid = null) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -150,7 +145,7 @@ exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
await db_api.removeAllRecords('archives', {sub_id: sub.id});
}
exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
if (typeof sub === 'string') {
// TODO: fix bad workaround where sub is a sub_id
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
@@ -201,13 +196,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
return false;
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
if (deleteForever) {
// ensure video is in the archives
const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
if (!exists_in_archive) {
await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
}
} else {
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
}
return true;
@@ -218,8 +208,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
}
}
exports.getVideosForSub = async (sub, user_uid = null) => {
const latest_sub_obj = await exports.getSubscription(sub.id);
async function getVideosForSub(sub, user_uid = null) {
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -239,20 +229,13 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
// remove temporary archive file if it exists
const archive_path = path.join(appendedBasePath, 'archive.txt');
const archive_exists = await fs.pathExists(archive_path);
if (archive_exists) {
await fs.unlink(archive_path);
}
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
@@ -307,7 +290,7 @@ async function handleOutputJSON(output, sub, user_uid) {
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
@@ -318,7 +301,7 @@ async function handleOutputJSON(output, sub, user_uid) {
return files_to_download;
}
exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -371,16 +354,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...qualityPath)
// skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
const archive_count = archive_text.split('\n').length - 1;
if (archive_count > 0) {
logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
const archive_path = path.join(appendedBasePath, 'archive.txt');
await fs.writeFile(archive_path, archive_text);
downloadConfig.push('--download-archive', archive_path);
}
if (sub.custom_args) {
const customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
@@ -435,8 +408,11 @@ async function getFilesToDownload(sub, output_jsons) {
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
}
files_to_download.push(output_json);
}
@@ -445,36 +421,35 @@ async function getFilesToDownload(sub, output_jsons) {
}
exports.getSubscriptions = async (user_uid = null) => {
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
exports.getAllSubscriptions = async () => {
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}
exports.getSubscription = async (subID) => {
async function getSubscription(subID) {
// stringify and parse because we may override the 'downloading' property
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
// now with the download_queue, we may need to override 'downloading'
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
return sub;
}
exports.getSubscriptionByName = async (subName, user_uid = null) => {
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
exports.updateSubscription = async (sub) => {
async function updateSubscription(sub) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
exports.writeSubscriptionMetadata(sub);
return true;
}
exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj);
});
@@ -486,14 +461,6 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
return true;
}
exports.writeSubscriptionMetadata = (sub) => {
let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
fs.writeJSONSync(metadata_path, sub);
}
async function setFreshUploads(sub) {
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
if (!sub_files) return;
@@ -548,3 +515,17 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
getSubscriptions : getSubscriptions,
getAllSubscriptions : getAllSubscriptions,
updateSubscription : updateSubscription,
subscribe : subscribe,
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
}

View File

@@ -2,18 +2,10 @@ const db_api = require('./db');
const notifications_api = require('./notifications');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
const subscriptions_api = require('./subscriptions');
const config_api = require('./config');
const auth_api = require('./authentication/auth');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const fs = require('fs-extra');
const path = require('path');
const logger = require('./logger');
const scheduler = require('node-schedule');
const { uuid } = require('uuidv4');
const TASKS = {
backup_local_db: {
@@ -28,7 +20,7 @@ const TASKS = {
job: null
},
missing_db_records: {
run: files_api.importUnregisteredFiles,
run: db_api.importUnregisteredFiles,
title: 'Import missing DB records',
job: null
},
@@ -54,11 +46,6 @@ const TASKS = {
run: archive_api.importArchives,
title: 'Import legacy archives',
job: null
},
rebuild_database: {
run: rebuildDB,
title: 'Rebuild database',
job: null
}
}
@@ -272,73 +259,9 @@ async function autoDeleteFiles(data) {
logger.info(`Removing ${data['files_to_remove'].length} old files!`);
for (let i = 0; i < data['files_to_remove'].length; i++) {
const file_to_remove = data['files_to_remove'][i];
await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
}
}
}
async function rebuildDB() {
await db_api.backupDB();
let subs_to_add = await guessSubscriptions(false);
subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
const users_to_add = await guessUsers();
for (const user_to_add of users_to_add) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_exists = await db_api.getRecord('users', {uid: user_to_add});
if (!user_exists) {
await auth_api.registerUser(user_to_add, user_to_add, 'password');
logger.info(`Regenerated user ${user_to_add}`);
}
const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
}
for (const sub_to_add of subs_to_add) {
const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
// TODO: we shouldn't be creating this here
const new_sub = Object.assign({}, sub_to_add, {paused: true});
if (!sub_exists) {
await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
logger.info(`Regenerated subscription ${sub_to_add['name']}`);
}
}
logger.info(`Importing unregistered files`);
await files_api.importUnregisteredFiles();
}
const guessUsers = async () => {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
return userPaths.map(userPath => path.basename(userPath));
}
const guessSubscriptions = async (isPlaylist, basePath = null) => {
const guessed_subs = [];
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
const subs = await utils.getDirectoriesInDirectory(subsPath);
for (const subPath of subs) {
const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
if (!fs.existsSync(sub_backup_path)) continue;
try {
const sub_backup = fs.readJSONSync(sub_backup_path)
delete sub_backup['_id'];
guessed_subs.push(sub_backup);
} catch(err) {
logger.warn(`Failed to reimport subscription in path ${subPath}`)
logger.warn(err);
}
}
return guessed_subs;
}
exports.TASKS = TASKS;

View File

@@ -1,4 +1,3 @@
/* eslint-disable no-undef */
const assert = require('assert');
const low = require('lowdb')
const winston = require('winston');
@@ -39,8 +38,6 @@ var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const archive_api = require('../archive');
const categories_api = require('../categories');
const files_api = require('../files');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
@@ -178,15 +175,6 @@ describe('Database', async function() {
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Update records', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test1'});
await db_api.insertRecordIntoTable('test', {test_update: 'test', key: 'test2'});
await db_api.updateRecords('test', {test_update: 'test'}, {added_field: true});
const updated_records = await db_api.getRecords('test', {added_field: true});
assert(updated_records.length === 2);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove property from record', async function() {
await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
@@ -337,56 +325,48 @@ describe('Database', async function() {
});
describe('Multi User', async function() {
const user_to_test = 'test_user';
const user_password = 'test_pass';
const sub_to_test = '';
const playlist_to_test = '';
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
await auth_api.deleteUser(user_to_test);
user = await auth_api.login('admin', 'pass');
});
describe('Basic', function() {
it('Register', async function() {
const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
assert(user);
});
it('Login', async function() {
await auth_api.registerUser(user_to_test, user_to_test, user_password);
const user = await auth_api.login(user_to_test, user_password);
describe('Authentication', function() {
it('login', async function() {
assert(user);
});
});
describe('Video player - normal', async function() {
beforeEach(async function() {
await db_api.removeRecord('files', {uid: sample_video_json['uid']});
await db_api.insertRecordIntoTable('files', sample_video_json);
});
await db_api.removeRecord('files', {uid: sample_video_json['uid']});
await db_api.insertRecordIntoTable('files', sample_video_json);
const video_to_test = sample_video_json['uid'];
it('Get video', async function() {
const video_obj = await files_api.getVideo(video_to_test);
const video_obj = await db_api.getVideo(video_to_test);
assert(video_obj);
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
const playlist_files_to_download = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const playlist_file = await files_api.getVideo(uid, user_to_test);
const playlist_file = await db_api.getVideo(uid, user_to_test);
playlist_files_to_download.push(playlist_file);
}
const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
@@ -414,7 +394,7 @@ describe('Multi User', async function() {
// const sub_to_test = '';
// const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
// it('Get video', async function() {
// const video_obj = files_api.getVideo(video_to_test, 'admin', );
// const video_obj = db_api.getVideo(video_to_test, 'admin', );
// assert(video_obj);
// });
@@ -517,23 +497,18 @@ describe('Downloader', function() {
const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
const updated_args1 = utils.injectArgs(original_args1, new_args1);
const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
assert(JSON.stringify(updated_args1) === JSON.stringify(expected_args1));
assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
const new_args2 = ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
const updated_args2 = utils.injectArgs(original_args2, new_args2);
const expected_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
assert(JSON.stringify(updated_args2) === JSON.stringify(expected_args2));
const original_args3 = ['-o', '%(title)s.%(ext)s'];
const new_args3 = ['--min-filesize','1'];
const updated_args3 = utils.injectArgs(original_args3, new_args3);
const expected_args3 = ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
const expected_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
console.log(updated_args2);
assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1710641401';
const example_vod = '1493770675';
it('Download VOD', async function() {
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
@@ -615,7 +590,7 @@ describe('Tasks', function() {
fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
await tasks_api.executeTask('missing_db_records');
const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
assert(!!imported_file === true);
assert(!!imported_file, true);
// post-test cleanup
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
@@ -753,109 +728,4 @@ describe('Utils', async function() {
assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
});
});
describe('Categories', async function() {
beforeEach(async function() {
await db_api.connectToDB();
const new_category = {
name: 'test_category',
uid: uuid(),
rules: [],
custom_output: ''
};
await db_api.insertRecordIntoTable('categories', new_category);
});
afterEach(async function() {
await db_api.removeAllRecords('categories', {name: 'test_category'});
});
it('Categorize - includes', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'includes',
property: 'title',
value: 'Sample'
});
const category = await categories_api.categorize([sample_video_json]);
assert(category && category.name === 'test_category');
});
it('Categorize - not includes', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'not_includes',
property: 'title',
value: 'Sample'
});
const category = await categories_api.categorize([sample_video_json]);
assert(!category);
});
it('Categorize - equals', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'equals',
property: 'uploader',
value: 'Sample Uploader'
});
const category = await categories_api.categorize([sample_video_json]);
console.log(category);
assert(category && category.name === 'test_category');
});
it('Categorize - not equals', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'not_equals',
property: 'uploader',
value: 'Sample Uploader'
});
const category = await categories_api.categorize([sample_video_json]);
assert(!category);
});
it('Categorize - AND', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'equals',
property: 'uploader',
value: 'Sample Uploader'
});
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: 'and',
comparator: 'not_includes',
property: 'title',
value: 'Sample'
});
const category = await categories_api.categorize([sample_video_json]);
assert(!category);
});
it('Categorize - OR', async function() {
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: null,
comparator: 'equals',
property: 'uploader',
value: 'Sample Uploader'
});
await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', {
preceding_operator: 'or',
comparator: 'not_includes',
property: 'title',
value: 'Sample'
});
const category = await categories_api.categorize([sample_video_json]);
assert(category);
});
});

View File

@@ -4,29 +4,19 @@ const logger = require('./logger');
const moment = require('moment');
const fs = require('fs-extra')
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
const commandExistsSync = require('command-exists').sync;
async function getCommentsForVOD(vodId) {
async function getCommentsForVOD(clientID, clientSecret, vodId) {
const { promisify } = require('util');
const child_process = require('child_process');
const exec = promisify(child_process.exec);
// Reject invalid params to prevent command injection attack
if (!vodId.match(/^[0-9a-z]+$/)) {
logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
return null;
}
const is_windows = process.platform === 'win32';
const cliExt = is_windows ? '.exe' : ''
const cliPath = `TwitchDownloaderCLI${cliExt}`
if (!commandExistsSync(cliPath)) {
logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
return null;
}
const result = await exec(`${cliPath} chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -83,7 +73,9 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const chat = await getCommentsForVOD(vodId);
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
// save file if needed params are included
let file_path = null;

View File

@@ -92,7 +92,7 @@ exports.createZipFile = async (zip_file_path, file_paths) => {
await archive.finalize();
// wait a tiny bit for the zip to reload in fs
await exports.wait(100);
await wait(100);
return zip_file_path;
}
@@ -414,11 +414,10 @@ exports.injectArgs = (original_args, new_args) => {
if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
if (original_args.includes(new_arg)) {
const original_index = original_args.indexOf(new_arg);
updated_args.splice(original_index, 2);
original_args.splice(original_index, 2);
}
updated_args.push(new_arg, new_args[i + 1]);
i++; // we need to skip the arg value on the next loop
} else {
if (!original_args.includes(new_arg)) {
updated_args.push(new_arg);
@@ -519,17 +518,6 @@ exports.convertFlatObjectToNestedObject = (obj) => {
return result;
}
exports.getDirectoriesInDirectory = async (basePath) => {
try {
const files = await fs.readdir(basePath, { withFileTypes: true });
return files
.filter((file) => file.isDirectory())
.map((file) => path.join(basePath, file.name));
} catch (err) {
return [];
}
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {

View File

@@ -21,4 +21,4 @@ version: 0.1.0
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "4.3.2"
appVersion: "4.3.1"

View File

@@ -18,7 +18,6 @@ services:
- "8998:17442"
image: tzahi12345/youtubedl-material:latest
ytdl-mongo-db:
# If you are using a Raspberry Pi, use mongo:4.4.18
image: mongo:4
logging:
driver: "none"

View File

@@ -1,69 +0,0 @@
import platform
import requests
import shutil
import os
import re
import sys
from collections import OrderedDict
from github import Github
machine = platform.machine()
# https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
MACHINES_TO_ZIP = OrderedDict([
("x86_64", "Linux-x64"),
("aarch64", "LinuxArm64"),
("armv8", "LinuxArm64"),
("arm", "LinuxArm"),
("AMD64", "Windows-x64")
])
def getZipName():
for possibleMachine, possibleZipName in MACHINES_TO_ZIP.items():
if possibleMachine in machine:
return possibleZipName
def getLatestFileInRepo(repo, search_string):
# Create an unauthenticated instance of the Github object
g = Github(os.environ.get('GH_TOKEN'))
# Replace with the repository owner and name
repo = g.get_repo(repo)
# Get all releases of the repository
releases = repo.get_releases()
# Loop through the releases in reverse order (from latest to oldest)
for release in list(releases):
# Get the release assets (files attached to the release)
assets = release.get_assets()
# Loop through the assets
for asset in assets:
if re.search(search_string, asset.name):
print(f'Downloading: {asset.name}')
response = requests.get(asset.browser_download_url)
with open(asset.name, 'wb') as f:
f.write(response.content)
print(f'Download complete: {asset.name}. Unzipping...')
shutil.unpack_archive(asset.name, './')
print(f'Unzipping complete!')
os.remove(asset.name)
break
else:
continue
break
else:
# If no matching release is found, print a message
print(f'No release found with {search_string}')
def getLatestCLIRelease():
zipName = getZipName()
if not zipName:
print(f"GetTwitchDownloader.py could not get valid path for '{machine}'. Exiting...")
sys.exit(1)
searchString = r'.*CLI.*' + zipName
getLatestFileInRepo("lay295/TwitchDownloader", searchString)
getLatestCLIRelease()

View File

@@ -1,39 +0,0 @@
#!/bin/sh
# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
# and also optimizing some code with this commit.
# xoxo :D
case $(uname -m) in
x86_64)
ARCH=Linux-x64;;
aarch64)
ARCH=LinuxArm64;;
armhf)
ARCH=LinuxArm;;
armv7)
ARCH=LinuxArm;;
armv7l)
ARCH=LinuxArm;;
*)
echo "Unsupported architecture: $(uname -m)"
exit 1
esac
echo "(INFO) Architecture detected: $ARCH"
echo "(1/5) READY - Install unzip"
apt-get update && apt-get -y install unzip curl jq libicu70
VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases" | jq -r --arg arch "$ARCH" '[.[] | select(.assets | length > 0) | select(.assets[].name | contains("CLI") and contains($arch))] | max_by(.published_at) | .tag_name')
echo "(2/5) DOWNLOAD - Acquire twitchdownloader"
curl -o twitchdownloader.zip \
--connect-timeout 5 \
--max-time 120 \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \
-L "https://github.com/lay295/TwitchDownloader/releases/download/$VERSION/TwitchDownloaderCLI-$VERSION-$ARCH.zip"
unzip twitchdownloader.zip
chmod +x TwitchDownloaderCLI
echo "(3/5) Smoke test"
./TwitchDownloaderCLI --help
cp ./TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI

View File

@@ -30,7 +30,7 @@ curl -o ffmpeg.txz \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \
"https://johnvansickle.com/ffmpeg/old-releases/ffmpeg-5.1.1-${ARCH}-static.tar.xz"
"https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
mkdir /tmp/ffmpeg
tar xf ffmpeg.txz -C /tmp/ffmpeg
echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"

41
main.js
View File

@@ -1,41 +0,0 @@
const { app, BrowserWindow } = require('electron');
const path = require('path');
const url = require('url');
let win;
function createWindow() {
win = new BrowserWindow({ width: 800, height: 600 });
// load the dist folder from Angular
win.loadURL(
url.format({
pathname: path.join(__dirname, `/dist/index.html`),
protocol: 'file:',
slashes: true
})
);
// The following is optional and will open the DevTools:
// win.webContents.openDevTools()
win.on('closed', () => {
win = null;
});
}
app.on('ready', createWindow);
// on macOS, closing the window doesn't quit the app
app.on('window-all-closed', () => {
if (process.platform !== 'darwin') {
app.quit();
}
});
// initialize the app's main window
app.on('activate', () => {
if (win === null) {
createWindow();
}
});

813
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "youtube-dl-material",
"version": "4.3.2",
"version": "4.3.1",
"license": "MIT",
"scripts": {
"ng": "ng",
@@ -34,19 +34,20 @@
"@angular/platform-browser-dynamic": "^15.0.1",
"@angular/router": "^15.0.1",
"@fontsource/material-icons": "^4.5.4",
"@ngneat/content-loader": "^7.0.0",
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^6.0.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
"filesize": "^10.0.7",
"filesize": "^6.1.0",
"fingerprintjs2": "^2.1.0",
"fs-extra": "^10.0.0",
"material-icons": "^1.10.8",
"nan": "^2.14.1",
"ngx-avatars": "^1.4.1",
"ngx-file-drop": "^15.0.0",
"ngx-file-drop": "^13.0.0",
"rxjs": "^6.6.3",
"rxjs-compat": "^6.6.7",
"rxjs-compat": "^6.0.0-rc.0",
"tslib": "^2.0.0",
"typescript": "~4.8.4",
"xliff-to-json": "^1.0.4",
@@ -59,7 +60,7 @@
"@angular/language-service": "^15.0.1",
"@types/core-js": "^2.5.2",
"@types/file-saver": "^2.0.1",
"@types/jasmine": "^4.3.1",
"@types/jasmine": "~3.6.0",
"@types/node": "^12.11.1",
"@typescript-eslint/eslint-plugin": "^4.29.0",
"@typescript-eslint/parser": "^4.29.0",
@@ -68,11 +69,11 @@
"eslint": "^7.32.0",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
"karma": "~6.4.2",
"karma": "~6.3.16",
"karma-chrome-launcher": "~3.1.0",
"karma-cli": "~1.0.1",
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~5.1.0",
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"openapi-typescript-codegen": "^0.23.0",
"protractor": "~7.0.0",

View File

@@ -104,7 +104,6 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
export type { SuccessObject } from './models/SuccessObject';
export type { TableInfo } from './models/TableInfo';
export type { Task } from './models/Task';
export { TaskType } from './models/TaskType';
export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
export type { TransferDBRequest } from './models/TransferDBRequest';

View File

@@ -5,4 +5,4 @@
export type AddFileToPlaylistRequest = {
file_uid: string;
playlist_id: string;
};
};

View File

@@ -13,4 +13,4 @@ export type Archive = {
sub_id?: string;
timestamp: number;
uid: string;
};
};

View File

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
export type BaseChangePermissionsRequest = {
permission: UserPermission;
new_value: YesNo;
};
};

View File

@@ -12,4 +12,4 @@ export type Category = {
* Overrides file output for downloaded files in category
*/
custom_output?: string;
};
};

View File

@@ -22,4 +22,4 @@ export namespace CategoryRule {
}
}
}

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
role: string;
});
});

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
user_uid: string;
});
});

View File

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
* UID of the concurrent stream
*/
uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
export type CheckConcurrentStreamResponse = {
stream: ConcurrentStream;
};
};

View File

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
clear_finished?: boolean;
clear_paused?: boolean;
clear_errors?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type Config = {
YoutubeDLMaterial: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { Config } from './Config';
export type ConfigResponse = {
config_file: Config;
success: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type CreateCategoryRequest = {
name: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Category } from './Category';
export type CreateCategoryResponse = {
new_category?: Category;
success?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
playlistName: string;
uids: Array<string>;
thumbnailURL: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
export type CreatePlaylistResponse = {
new_playlist: Playlist;
success: boolean;
};
};

View File

@@ -5,4 +5,4 @@
export type CropFileSettings = {
cropFileStart: number;
cropFileEnd: number;
};
};

View File

@@ -17,4 +17,4 @@ export namespace DBBackup {
}
}
}

View File

@@ -16,4 +16,4 @@ roles?: TableInfo;
download_queue?: TableInfo;
archives?: TableInfo;
};
};
};

View File

@@ -42,4 +42,4 @@ export type DatabaseFile = {
*/
abr?: number;
favorite: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
* Number of files removed
*/
delete_count?: number;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type DeleteArchiveItemsRequest = {
archives: Array<Archive>;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteCategoryRequest = {
category_uid: string;
};
};

View File

@@ -5,4 +5,4 @@
export type DeleteMp3Mp4Request = {
uid: string;
blacklistMode?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteNotificationRequest = {
uid: string;
};
};

View File

@@ -4,4 +4,4 @@
export type DeletePlaylistRequest = {
playlist_id: string;
};
};

View File

@@ -8,4 +8,4 @@ export type DeleteSubscriptionFileRequest = {
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteUserRequest = {
uid: string;
};
};

View File

@@ -27,4 +27,4 @@ export type Download = {
sub_id?: string;
sub_name?: string;
prefetched_info?: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type DownloadArchiveRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
playlist_id?: string;
url?: string;
type?: FileType;
};
};

View File

@@ -49,4 +49,4 @@ export type DownloadRequest = {
* If using youtube-dl archive, download will ignore it
*/
ignoreArchive?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type DownloadResponse = {
download?: Download;
};
};

View File

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
export type DownloadTwitchChatByVODIDResponse = {
chat: Array<TwitchChatMessage>;
};
};

View File

@@ -4,4 +4,4 @@
export type DownloadVideosForSubscriptionRequest = {
subID: string;
};
};

View File

@@ -5,4 +5,4 @@
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}
}

View File

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
AUDIO_ONLY = 'audio_only',
VIDEO_ONLY = 'video_only',
BOTH = 'both',
}
}

View File

@@ -4,4 +4,4 @@
export type GenerateArgsResponse = {
args?: Array<string>;
};
};

View File

@@ -4,4 +4,4 @@
export type GenerateNewApiKeyResponse = {
new_api_key: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Category } from './Category';
export type GetAllCategoriesResponse = {
categories: Array<Category>;
};
};

View File

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
* Filters downloads with the array
*/
uids?: Array<string> | null;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetAllDownloadsResponse = {
downloads?: Array<Download>;
};
};

View File

@@ -21,4 +21,4 @@ export type GetAllFilesRequest = {
* Include if you want to filter by subscription
*/
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
export type GetAllSubscriptionsResponse = {
subscriptions: Array<Subscription>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Task } from './Task';
export type GetAllTasksResponse = {
tasks?: Array<Task>;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type GetArchivesRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type GetArchivesResponse = {
archives: Array<Archive>;
};
};

View File

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
export type GetDBBackupsResponse = {
tasks?: Array<DBBackup>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetDownloadRequest = {
download_uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetDownloadResponse = {
download?: Download;
};
};

View File

@@ -4,4 +4,4 @@
export type GetFileFormatsRequest = {
url?: string;
};
};

View File

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
result: {
formats?: Array<any>;
};
};
};

View File

@@ -14,4 +14,4 @@ export type GetFileRequest = {
* User UID
*/
uuid?: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { DatabaseFile } from './DatabaseFile';
export type GetFileResponse = {
success: boolean;
file?: DatabaseFile;
};
};

View File

@@ -16,4 +16,4 @@ export type GetFullTwitchChatRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -5,4 +5,4 @@
export type GetFullTwitchChatResponse = {
success: boolean;
error?: string;
};
};

View File

@@ -4,4 +4,4 @@
export type GetLogsRequest = {
lines?: number;
};
};

View File

@@ -8,4 +8,4 @@ export type GetLogsResponse = {
*/
logs?: string;
success?: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type GetMp3sResponse = {
* All audio playlists
*/
playlists: Array<Playlist>;
};
};

Some files were not shown because too many files have changed in this diff Show More