Compare commits

..

31 Commits

Author SHA1 Message Date
Isaac Abadi
5d79f966e1 Merge branch 'gh-pages' of https://github.com/Tzahi12345/YoutubeDL-Material into gh-pages 2023-04-30 13:05:55 -04:00
Isaac Abadi
c3bd642c1a Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into gh-pages 2023-04-30 13:05:36 -04:00
Tzahi12345
872e99021f Delete CNAME 2023-04-30 11:29:51 -04:00
Tzahi12345
74e1c73f64 Create CNAME 2023-04-30 01:42:32 -04:00
Tzahi12345
71ca2adac2 Delete CNAME 2023-04-30 01:42:24 -04:00
Tzahi12345
b35aab9509 Create CNAME 2023-04-30 01:01:09 -04:00
Tzahi12345
e280f4af60 Delete CNAME 2023-04-30 01:00:41 -04:00
Tzahi12345
6276df99f8 Create CNAME 2023-04-29 21:50:51 -04:00
Tzahi12345
bd93650517 Delete CNAME 2023-04-29 21:50:40 -04:00
Tzahi12345
110bae3e5b Create CNAME 2023-04-29 21:43:31 -04:00
Isaac Abadi
8c6e583e1a Updated chart location, renamed yamls and hardcoded service name in ingress 2021-01-01 05:54:52 -05:00
Isaac Abadi
9ee2599cb2 Claim sizes are now configurable 2020-12-31 17:02:18 -05:00
Isaac Abadi
9306375098 Made selector label static 2020-12-31 16:51:13 -05:00
Isaac Abadi
4fb2b87302 Testing static port 2020-12-31 16:41:18 -05:00
Isaac Abadi
75b081603f Fixed bug in deployment of chart and merged two service files 2020-12-31 16:29:22 -05:00
Isaac Abadi
06636fa2cb Added helper tpl to define variables used in template 2020-12-31 16:19:00 -05:00
Isaac Abadi
9d6887a92c Trying new names for chart in template 2020-12-31 15:32:07 -05:00
Isaac Abadi
85e2958be3 Added ingress and expandable storage support 2020-12-31 15:22:53 -05:00
Isaac Abadi
c67d5c3734 Added license, updated audio claim, and updated archive for chart 2020-12-31 15:08:15 -05:00
Isaac Abadi
a42e8b6554 Fixed bug in name of subscriptions claim 2020-12-31 14:50:22 -05:00
Isaac Abadi
5ba3028665 Added icon and additional info to chart.yaml 2020-12-31 06:25:11 -05:00
Isaac Abadi
d25240acc7 Updated chart description and fixed typo that caused deployment of pod to fail 2020-12-31 06:12:47 -05:00
Isaac Abadi
8d54e1d643 Updated chart README 2020-12-31 05:59:25 -05:00
Isaac Abadi
fea9167fb1 Increased size of claims further and removed old claims 2020-12-31 05:42:34 -05:00
Isaac Abadi
9e4fac3b43 Renamed claims and increased size 2020-12-31 05:38:39 -05:00
Isaac Abadi
f748574417 Updated chart archive 2020-12-31 02:01:58 -05:00
Isaac Abadi
7c18d3d69d Updated chart to use persistent volumes 2020-12-31 01:58:31 -05:00
Isaac Abadi
391b8286e9 Updated charts archive 2020-12-30 20:32:33 -05:00
Isaac Abadi
f44ad92a08 Updated name for chart/index 2020-12-30 20:06:46 -05:00
Isaac Abadi
5730b40459 Added artifacthub repo metadata file 2020-12-30 20:03:35 -05:00
Isaac Abadi
5a7b6243fd Added helm chart 2020-12-30 19:31:36 -05:00
206 changed files with 2232 additions and 2944 deletions

View File

@@ -15,9 +15,9 @@ jobs:
- name: checkout code
uses: actions/checkout@v2
- name: setup node
uses: actions/setup-node@v3
uses: actions/setup-node@v2
with:
node-version: '16'
node-version: '14'
cache: 'npm'
- name: install dependencies
run: |
@@ -33,7 +33,7 @@ jobs:
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "autobuild", "tag": "N/A", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -55,7 +55,7 @@ jobs:
Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material
Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material
- name: upload build artifact
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v1
with:
name: youtubedl-material
path: build

View File

@@ -18,21 +18,10 @@ jobs:
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
dir: 'backend/'
- name: setup platform emulator
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v1
- name: build & push images
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
#platforms: linux/amd64
push: false
tags: tzahi12345/youtubedl-material:nightly-pr
- name: Build docker images
run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

View File

@@ -27,7 +27,7 @@ jobs:
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "latest", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -60,10 +60,10 @@ jobs:
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64/v8
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}

View File

@@ -34,7 +34,7 @@ jobs:
- name: create-json
id: create-json
uses: jsdaniell/create-json@v1.2.2
uses: jsdaniell/create-json@1.1.2
with:
name: "version.json"
json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
@@ -44,7 +44,7 @@ jobs:
uses: docker/setup-qemu-action@v1
- name: setup multi-arch docker build
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Generate Docker image metadata
id: docker-meta
@@ -63,7 +63,7 @@ jobs:
type=sha,prefix=sha-,format=short
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
platforms: linux/amd64,linux/arm,linux/arm64/v8
push: true
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}

14
.vscode/launch.json vendored
View File

@@ -4,20 +4,6 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Dev: Debug Backend",
"request": "launch",
"runtimeArgs": [
"run-script",
"debug"
],
"runtimeExecutable": "npm",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"cwd": "${workspaceFolder}/backend"
},
{
"type": "node",
"request": "attach",

View File

@@ -1,38 +0,0 @@
<h1>Development</h1>
- [First time...](#first-time)
- [Setup](#setup)
- [Startup](#startup)
- [Debugging the backend (VSC)](#debugging-the-backend-vsc)
- [Deploy changes](#deploy-changes)
- [Frontend](#frontend)
- [Backend](#backend)
# First time...
## Setup
Checkout the repository and navigate to the `youtubedl-material` directory.
```bash
vim ./src/assets/default.json # Edit settings for your local environment. This config file is just the dev config file, if YTDL_MODE is not set to "debug", then ./backend/appdata/default.json will be used
npm -g install pm2 # Install pm2
npm install # Install dependencies for the frontend
cd ./backend
npm install # Install dependencies for the backend
cd ..
npm run build # Build the frontend
```
This step have to be done only once.
## Startup
Navigate to the `youtubedl-material/backend` directory and run `npm start`.
# Debugging the backend (VSC)
Open the `youtubedl-material` directory in Visual Studio Code and run the launch configuration `Dev: Debug Backend`.
# Deploy changes
## Frontend
Navigate to the `youtubedl-material` directory and run `npm run build`. Restart the backend.
## Backend
Simply restart the backend.

View File

@@ -1,17 +1,15 @@
# Fetching our utils
FROM ubuntu:22.04 AS utils
# Fetching our ffmpeg
FROM ubuntu:22.04 AS ffmpeg
ENV DEBIAN_FRONTEND=noninteractive
# Use script due local build compability
COPY docker-utils/*.sh .
RUN chmod +x *.sh
COPY docker-utils/ffmpeg-fetch.sh .
RUN chmod +x ffmpeg-fetch.sh
RUN sh ./ffmpeg-fetch.sh
RUN sh ./fetch-twitchdownloader.sh
# Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021)
# Go to 20.04
FROM ubuntu:22.04 AS base
ARG TARGETPLATFORM
FROM ubuntu:20.04 AS base
ARG DEBIAN_FRONTEND=noninteractive
ENV UID=1000
ENV GID=1000
@@ -19,30 +17,19 @@ ENV USER=youtube
ENV NO_UPDATE_NOTIFIER=true
ENV PM2_HOME=/app/pm2
ENV ALLOW_CONFIG_MUTATIONS=true
# Directy fetch specific version
## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb
RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \
apt update && \
apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 && \
apt install -y --no-install-recommends curl ca-certificates tzdata && \
curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
apt install -y --no-install-recommends nodejs && \
npm -g install npm n && \
n 16.14.2 && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN case ${TARGETPLATFORM} in \
"linux/amd64") NODE_ARCH=amd64 ;; \
"linux/arm") NODE_ARCH=armhf ;; \
"linux/arm/v7") NODE_ARCH=armhf ;; \
"linux/arm64") NODE_ARCH=arm64 ;; \
esac \
&& curl -L https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_$NODE_ARCH.deb -o ./nodejs.deb && \
apt update && \
apt install -y ./nodejs.deb && \
apt clean && \
rm -rf /var/lib/apt/lists/* &&\
rm nodejs.deb;
# Build frontend
ARG BUILDPLATFORM
FROM --platform=${BUILDPLATFORM} node:16 as frontend
FROM base as frontend
RUN npm install -g @angular/cli
WORKDIR /build
COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ]
@@ -50,8 +37,6 @@ COPY [ "src/", "/build/src/" ]
RUN npm install && \
npm run build && \
ls -al /build/backend/public
RUN npm uninstall -g @angular/cli
RUN rm -rf node_modules
# Install backend deps
@@ -62,35 +47,31 @@ RUN npm config set strict-ssl false && \
npm install --prod && \
ls -al
#FROM base as python
# armv7 need build from source
#WORKDIR /app
#COPY docker-utils/GetTwitchDownloader.py .
#RUN apt update && \
# apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip python3-dev build-essential libffi-dev && \
# apt clean && \
# rm -rf /var/lib/apt/lists/*
#RUN pip install PyGithub requests
#RUN python GetTwitchDownloader.py
FROM base as python
WORKDIR /app
COPY docker-utils/GetTwitchDownloader.py .
RUN apt update && \
apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install PyGithub requests
RUN python GetTwitchDownloader.py
# Final image
FROM base
RUN npm install -g pm2 && \
apt update && \
apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
pip install pycryptodomex && \
apt remove -y --purge build-essential && \
apt autoremove -y --purge && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install pycryptodomex
WORKDIR /app
# User 1000 already exist from base image
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/usr/local/bin/TwitchDownloaderCLI"]
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ]
COPY --chown=$UID:$GID --from=backend ["/app/","/app/"]
COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
#COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"]
RUN chown $UID:$GID .
RUN chmod +x /app/fix-scripts/*.sh
# Add some persistence data
#VOLUME ["/app/appdata"]

View File

@@ -1758,14 +1758,14 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
required:
- task_key
UpdateTaskScheduleRequest:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_schedule:
$ref: '#/components/schemas/Schedule'
required:
@@ -1775,7 +1775,7 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_data:
type: object
required:
@@ -1785,7 +1785,7 @@ components:
type: object
properties:
task_key:
$ref: '#/components/schemas/TaskType'
type: string
new_options:
type: object
required:
@@ -2726,7 +2726,7 @@ components:
type: object
properties:
key:
$ref: '#/components/schemas/TaskType'
type: string
title:
type: string
last_ran:
@@ -2742,20 +2742,9 @@ components:
error:
type: string
schedule:
$ref: '#/components/schemas/Schedule'
type: object
options:
type: object
TaskType:
type: string
enum:
- backup_local_db
- missing_files_check
- missing_db_records
- duplicate_files_check
- youtubedl_update_check
- delete_old_files
- import_legacy_archives
- rebuild_database
Schedule:
required:
- type
@@ -2888,7 +2877,6 @@ components:
- sharing
- advanced_download
- downloads_manager
- tasks_manager
YesNo:
type: string
enum:

View File

@@ -28,28 +28,13 @@ Dark mode:
NOTE: If you would like to use Docker, you can skip down to the [Docker](#Docker) section for a setup guide.
Required dependencies:
* Node.js 16
* Python
Optional dependencies:
* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
* [Twitch Downloader CLI](https://github.com/lay295/TwitchDownloader) (for downloading Twitch VOD chats)
<details>
<summary>Debian/Ubuntu</summary>
Debian/Ubuntu:
```bash
curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
sudo apt-get install nodejs youtube-dl ffmpeg unzip python npm
```
</details>
<details>
<summary>CentOS 7</summary>
CentOS 7:
```bash
sudo yum install epel-release
@@ -57,11 +42,13 @@ sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfu
sudo yum install centos-release-scl-rh
sudo yum install rh-nodejs12
scl enable rh-nodejs12 bash
curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash -
sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
```
</details>
Optional dependencies:
* AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
### Installing
@@ -85,9 +72,7 @@ If you'd like to install YoutubeDL-Material, go to the Installation section. If
To deploy, simply clone the repository, and go into the `youtubedl-material` directory. Type `npm install` and all the dependencies will install. Then type `cd backend` and again type `npm install` to install the dependencies for the backend.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm run build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
Lastly, type `npm -g install pm2` to install pm2 globally.
Once you do that, you're almost up and running. All you need to do is edit the configuration in `youtubedl-material/appdata`, go back into the `youtubedl-material` directory, and type `npm build`. This will build the app, and put the output files in the `youtubedl-material/backend/public` folder.
The frontend is now complete. The backend is much easier. Just go into the `backend` folder, and type `npm start`.

View File

@@ -2,6 +2,7 @@ const { uuid } = require('uuidv4');
const fs = require('fs-extra');
const { promisify } = require('util');
const auth_api = require('./authentication/auth');
const winston = require('winston');
const path = require('path');
const compression = require('compression');
const multer = require('multer');
@@ -18,7 +19,6 @@ const CONSTS = require('./consts')
const read_last_lines = require('read-last-lines');
const ps = require('ps-node');
const Feed = require('feed').Feed;
const session = require('express-session');
// needed if bin/details somehow gets deleted
if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -34,7 +34,6 @@ const categories_api = require('./categories');
const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
var app = express();
@@ -163,7 +162,6 @@ app.use(bodyParser.json());
// use passport
app.use(auth_api.passport.initialize());
app.use(session({ secret: uuid(), resave: true, saveUninitialized: true }))
app.use(auth_api.passport.session());
// actual functions
@@ -175,10 +173,10 @@ async function checkMigrations() {
if (!simplified_db_migration_complete) {
logger.info('Beginning migration: 4.1->4.2+')
let success = await simplifyDBFileStructure();
success = success && await files_api.addMetadataPropertyToDB('view_count');
success = success && await files_api.addMetadataPropertyToDB('description');
success = success && await files_api.addMetadataPropertyToDB('height');
success = success && await files_api.addMetadataPropertyToDB('abr');
success = success && await db_api.addMetadataPropertyToDB('view_count');
success = success && await db_api.addMetadataPropertyToDB('description');
success = success && await db_api.addMetadataPropertyToDB('height');
success = success && await db_api.addMetadataPropertyToDB('abr');
// sets migration to complete
db.set('simplified_db_migration_complete', true).write();
if (success) { logger.info('4.1->4.2+ migration complete!'); }
@@ -535,7 +533,6 @@ async function loadConfig() {
if (allowSubscriptions) {
// set downloading to false
let subscriptions = await subscriptions_api.getAllSubscriptions();
subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false});
// runs initially, then runs every ${subscriptionCheckInterval} seconds
const watchSubscriptionsInterval = function() {
@@ -727,7 +724,7 @@ const optionalJwt = async function (req, res, next) {
const uuid = using_body ? req.body.uuid : req.query.uuid;
const uid = using_body ? req.body.uid : req.query.uid;
const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
if (file) {
req.can_watch = true;
return next();
@@ -938,7 +935,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
const sub_id = req.body.sub_id;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
res.send({
files: files,
@@ -1104,7 +1101,7 @@ app.post('/api/incrementViewCount', async (req, res) => {
uuid = req.user.uid;
}
const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
const file_obj = await db_api.getVideo(file_uid, uuid, sub_id);
const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
const new_view_count = current_view_count + 1;
@@ -1232,7 +1229,7 @@ app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
let deleteForever = req.body.deleteForever;
let file_uid = req.body.file_uid;
let success = await files_api.deleteFile(file_uid, deleteForever);
let success = await db_api.deleteFile(file_uid, deleteForever);
if (success) {
res.send({
@@ -1320,7 +1317,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
let playlistName = req.body.playlistName;
let uids = req.body.uids;
const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
res.send({
new_playlist: new_playlist,
@@ -1333,13 +1330,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
let include_file_metadata = req.body.include_file_metadata;
const playlist = await files_api.getPlaylist(playlist_id, uuid);
const playlist = await db_api.getPlaylist(playlist_id, uuid);
const file_objs = [];
if (playlist && include_file_metadata) {
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await files_api.getVideo(uid, uuid);
const file_obj = await db_api.getVideo(uid, uuid);
if (file_obj) file_objs.push(file_obj);
// TODO: remove file from playlist if could not be found
}
@@ -1377,7 +1374,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
playlist.uids.push(file_uid);
let success = await files_api.updatePlaylist(playlist);
let success = await db_api.updatePlaylist(playlist);
res.send({
success: success
});
@@ -1385,7 +1382,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
let playlist = req.body.playlist;
let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
res.send({
success: success
});
@@ -1415,7 +1412,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => {
const blacklistMode = req.body.blacklistMode;
let wasDeleted = false;
wasDeleted = await files_api.deleteFile(uid, blacklistMode);
wasDeleted = await db_api.deleteFile(uid, blacklistMode);
res.send(wasDeleted);
});
@@ -1447,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
for (let i = 0; i < files.length; i++) {
let wasDeleted = false;
wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode);
wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
if (wasDeleted) {
delete_count++;
}
@@ -1473,10 +1470,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
if (playlist_id) {
zip_file_generated = true;
const playlist_files_to_download = [];
const playlist = await files_api.getPlaylist(playlist_id, uuid);
const playlist = await db_api.getPlaylist(playlist_id, uuid);
for (let i = 0; i < playlist['uids'].length; i++) {
const playlist_file_uid = playlist['uids'][i];
const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
playlist_files_to_download.push(file_obj);
}
@@ -1490,7 +1487,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
// generate zip
file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
} else {
const file_obj = await files_api.getVideo(uid, uuid, sub_id)
const file_obj = await db_api.getVideo(uid, uuid, sub_id)
file_path_to_download = file_obj.path;
}
if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
@@ -1637,7 +1634,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
file_obj = await files_api.getVideo(uid, uuid, sub_id);
file_obj = await db_api.getVideo(uid, uuid, sub_id);
if (file_obj) file_path = file_obj['path'];
else file_path = null;
}
@@ -1929,38 +1926,9 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
// user authentication
app.post('/api/auth/register', optionalJwt, async (req, res) => {
const userid = req.body.userid;
const username = req.body.username;
const plaintextPassword = req.body.password;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
res.sendStatus(409);
return;
}
if (plaintextPassword === "") {
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
res.sendStatus(409);
return;
}
if (!userid || !username) {
logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
}
const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
if (!new_user) {
res.sendStatus(409);
return;
}
res.send({
user: new_user
});
});
app.post('/api/auth/register'
, optionalJwt
, auth_api.registerUser);
app.post('/api/auth/login'
, auth_api.passport.authenticate(['local', 'ldapauth'], {})
, auth_api.generateJWT
@@ -2012,7 +1980,18 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => {
app.post('/api/deleteUser', optionalJwt, async (req, res) => {
let uid = req.body.uid;
try {
const success = await auth_api.deleteUser(uid);
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
}
res.send({success: success});
} catch (err) {
logger.error(err);
@@ -2103,7 +2082,7 @@ app.get('/api/rss', async function (req, res) {
const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const feed = new Feed({
title: 'Downloads',

View File

@@ -50,8 +50,7 @@
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": ""
"discord_webhook_URL": ""
},
"Themes": {
"default_theme": "default",

View File

@@ -1,13 +1,11 @@
const config_api = require('../config');
const CONSTS = require('../consts');
const consts = require('../consts');
const logger = require('../logger');
const db_api = require('../db');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
const bcrypt = require('bcryptjs');
const fs = require('fs-extra');
const path = require('path');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -18,7 +16,7 @@ var JwtStrategy = require('passport-jwt').Strategy,
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = 10;
let saltRounds = null;
exports.initialize = function () {
/*************************
@@ -33,6 +31,8 @@ exports.initialize = function () {
});
}
saltRounds = 10;
// Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
if (!(+JWT_EXPIRATION)) {
@@ -68,7 +68,14 @@ exports.initialize = function () {
const setupRoles = async () => {
const required_roles = {
admin: {
permissions: CONSTS.AVAILABLE_PERMISSIONS
permissions: [
'filemanager',
'settings',
'subscriptions',
'sharing',
'advanced_download',
'downloads_manager'
]
},
user: {
permissions: [
@@ -106,41 +113,55 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
exports.registerUser = async (userid, username, plaintextPassword) => {
const hash = await bcrypt.hash(plaintextPassword, saltRounds);
const new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
return null;
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
return null;
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
return new_user;
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
res.sendStatus(409);
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
return;
}
}
exports.deleteUser = async (uid) => {
let success = false;
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_folder = path.join(__dirname, usersFileFolder, uid);
const user_db_obj = await db_api.getRecord('users', {uid: uid});
if (user_db_obj) {
// user exists, let's delete
await fs.remove(user_folder);
await db_api.removeRecord('users', {uid: uid});
success = true;
} else {
logger.error(`Could not find user with uid ${uid}`);
if (plaintextPassword === "") {
res.sendStatus(400);
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
return;
}
return success;
bcrypt.hash(plaintextPassword, saltRounds)
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
});
}
})
.then(function(result) {
})
.catch(function(err) {
logger.error(err);
if( err.code == 'ER_DUP_ENTRY' ) {
res.status(409).send('UserId already taken');
} else {
res.sendStatus(409);
}
});
}
/***************************************
@@ -221,7 +242,7 @@ exports.returnAuthResponse = async function(req, res) {
user: req.user,
token: req.token,
permissions: await exports.userPermissions(req.user.uid),
available_permissions: CONSTS.AVAILABLE_PERMISSIONS
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -305,7 +326,7 @@ exports.getUserVideos = async function(user_uid, type) {
}
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {uid: file_uid});
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -392,8 +413,8 @@ exports.userPermissions = async function(user_uid) {
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) {
let permission = CONSTS.AVAILABLE_PERMISSIONS[i];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);

View File

@@ -220,8 +220,7 @@ const DEFAULT_CONFIG = {
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": "",
"discord_webhook_URL": ""
},
"Themes": {
"default_theme": "default",

View File

@@ -162,10 +162,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_discord_webhook_url',
'path': 'YoutubeDLMaterial.API.discord_webhook_URL'
},
'ytdl_slack_webhook_url': {
'key': 'ytdl_slack_webhook_url',
'path': 'YoutubeDLMaterial.API.slack_webhook_URL'
},
// Themes
@@ -347,8 +343,6 @@ const YTDL_ARGS_WITH_VALUES = [
'--convert-subs'
];
exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json'
// we're using a Set here for performance
exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);

View File

@@ -1,11 +1,11 @@
const fs = require('fs-extra')
const path = require('path')
var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config');
const utils = require('./utils')
var utils = require('./utils')
const logger = require('./logger');
const low = require('lowdb')
@@ -167,9 +167,82 @@ exports._connectToDB = async (custom_connection_string = null) => {
}
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
// TODO: check if video exists, throw error if not
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']})
return file_object;
}
function generateFileObject(file_path, type) {
var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
} else if (!jsonobj['_filename']) {
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
var stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = utils.formatDateString(jsonobj.upload_date);
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function getAppendedBasePathSub(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
exports.getFileDirectoriesAndDBs = async () => {
@@ -244,6 +317,277 @@ exports.getFileDirectoriesAndDBs = async () => {
return dirs_to_check;
}
exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it
const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) {
// add additional info
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
}
}
}
return imported_files;
}
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await exports.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
const first_video = await exports.getVideo(uids[0]);
const thumbnailToUse = first_video['thumbnailURL'];
let new_playlist = {
name: playlist_name,
uids: uids,
id: uuid(),
thumbnailURL: thumbnailToUse,
registered: Date.now(),
randomize_order: false
};
new_playlist.user_uid = user_uid ? user_uid : undefined;
await exports.insertRecordIntoTable('playlists', new_playlist);
const duration = await exports.calculatePlaylistDuration(new_playlist);
await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = await exports.getRecord('playlists', {id: playlist_id});
if (!playlist) {
playlist = await exports.getRecord('categories', {uid: playlist_id});
if (playlist) {
const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
playlist['uids'] = uids;
playlist['auto'] = true;
}
}
// converts playlists to new UID-based schema
if (playlist && playlist['fileNames'] && !playlist['uids']) {
playlist['uids'] = [];
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i];
const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
}
exports.updatePlaylist(playlist, user_uid);
}
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
return playlist;
}
exports.updatePlaylist = async (playlist) => {
let playlistID = playlist.id;
const duration = await exports.calculatePlaylistDuration(playlist);
playlist.duration = duration;
return await exports.updateRecord('playlists', {id: playlistID}, playlist);
}
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj);
}
if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
}
exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
if (!playlist_file_objs) {
playlist_file_objs = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await exports.getVideo(uid);
if (file_obj) playlist_file_objs.push(file_obj);
}
}
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[uid]) {
try {
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
config_api.descriptors[uid][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
// get id/extractor from JSON
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
await exports.removeRecord('files', {uid: uid});
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = async (file_id, uuid = null) => {
const file_obj = await exports.getRecord('files', {id: file_id});
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid) => {
return await exports.getRecord('files', {uid: file_uid});
}
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await exports.getRecords('files', filter_obj, true);
return {files, file_count};
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
// TODO: check if video exists, throw error if not
await exports.updateRecord('files', {uid: file_uid}, assignment_obj);
}
// Basic DB functions
// Create

View File

@@ -13,7 +13,6 @@ const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const files_api = require('./files');
const notifications_api = require('./notifications');
const archive_api = require('./archive');
@@ -222,7 +221,6 @@ async function collectInfo(download_uid) {
return;
}
// in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
@@ -386,9 +384,10 @@ async function downloadQueuedFile(download_uid) {
}
// registers file in DB
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
@@ -400,7 +399,7 @@ async function downloadQueuedFile(download_uid) {
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {

View File

@@ -1,7 +1,17 @@
#!/bin/bash
#!/bin/sh
set -eu
CMD="npm start"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then
set -- "$CMD" "$@"
fi
# chown current working directory to current user
echo "[entrypoint] setup permission, this may take a while"
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$@"
if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$0" "$@"
fi
exec "$@"

View File

@@ -1,350 +0,0 @@
const fs = require('fs-extra')
const path = require('path')
const { uuid } = require('uuidv4');
const config_api = require('./config');
const db_api = require('./db');
const archive_api = require('./archive');
const utils = require('./utils')
const logger = require('./logger');
exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject(file_path, type);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false;
}
utils.fixVideoMetadataPerms(file_path, type);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (user_uid) file_object['user_uid'] = user_uid;
if (sub_id) file_object['sub_id'] = sub_id;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_path, type)
}
return file_obj;
}
async function registerFileDBManual(file_object) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
const path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']})
return file_object;
}
function generateFileObject(file_path, type) {
const jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) {
return null;
} else if (!jsonobj['_filename']) {
logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`);
return null;
}
const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type);
// console.
const stats = fs.statSync(true_file_path);
const file_id = utils.removeFileExtension(path.basename(file_path));
const title = jsonobj.title;
const url = jsonobj.webpage_url;
const uploader = jsonobj.uploader;
const upload_date = utils.formatDateString(jsonobj.upload_date);
const size = stats.size;
const thumbnail = jsonobj.thumbnail;
const duration = jsonobj.duration;
const isaudio = type === 'audio';
const description = jsonobj.description;
const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
for (let j = 0; j < files.length; j++) {
const file = files[j];
// check if file exists in db, if not add it
const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id});
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) {
// add additional info
const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
}
}
}
return imported_files;
}
exports.addMetadataPropertyToDB = async (property_key) => {
try {
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
const update_obj = {};
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
// recursively get all files in dir's path
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true);
for (let j = 0; j < files.length; j++) {
const file = files[j];
if (file[property_key]) {
update_obj[file.uid] = {[property_key]: file[property_key]};
}
}
}
return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
}
}
exports.createPlaylist = async (playlist_name, uids, user_uid = null) => {
const first_video = await exports.getVideo(uids[0]);
const thumbnailToUse = first_video['thumbnailURL'];
let new_playlist = {
name: playlist_name,
uids: uids,
id: uuid(),
thumbnailURL: thumbnailToUse,
registered: Date.now(),
randomize_order: false
};
new_playlist.user_uid = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('playlists', new_playlist);
const duration = await exports.calculatePlaylistDuration(new_playlist);
await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration});
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = await db_api.getRecord('playlists', {id: playlist_id});
if (!playlist) {
playlist = await db_api.getRecord('categories', {uid: playlist_id});
if (playlist) {
const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid);
playlist['uids'] = uids;
playlist['auto'] = true;
}
}
// converts playlists to new UID-based schema
if (playlist && playlist['fileNames'] && !playlist['uids']) {
playlist['uids'] = [];
logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`);
for (let i = 0; i < playlist['fileNames'].length; i++) {
const fileName = playlist['fileNames'][i];
const uid = await exports.getVideoUIDByID(fileName, user_uid);
if (uid) playlist['uids'].push(uid);
else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
}
exports.updatePlaylist(playlist, user_uid);
}
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
return playlist;
}
exports.updatePlaylist = async (playlist) => {
let playlistID = playlist.id;
const duration = await exports.calculatePlaylistDuration(playlist);
playlist.duration = duration;
return await db_api.updateRecord('playlists', {id: playlistID}, playlist);
}
exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => {
let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj);
if (!success) {
success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj);
}
if (!success) {
logger.error(`Could not find playlist or category with ID ${playlist_id}`);
}
return success;
}
exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => {
if (!playlist_file_objs) {
playlist_file_objs = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await exports.getVideo(uid);
if (file_obj) playlist_file_objs.push(file_obj);
}
}
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[uid]) {
try {
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
config_api.descriptors[uid][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive || file_obj.sub_id) {
// get id/extractor from JSON
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
if (!blacklistMode) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id)
} else {
const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
if (!exists_in_archive) {
await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id);
}
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
await db_api.removeRecord('files', {uid: uid});
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = async (file_id, uuid = null) => {
const file_obj = await db_api.getRecord('files', {id: file_id});
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid) => {
return await db_api.getRecord('files', {uid: file_uid});
}
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await db_api.getRecords('files', filter_obj, true);
return {files, file_count};
}

View File

@@ -64,9 +64,6 @@ exports.sendNotification = async (notification) => {
if (config_api.getConfigItem('ytdl_discord_webhook_url')) {
sendDiscordNotification(data);
}
if (config_api.getConfigItem('ytdl_slack_webhook_url')) {
sendSlackNotification(data);
}
await db_api.insertRecordIntoTable('notifications', notification);
return notification;
@@ -177,65 +174,6 @@ async function sendDiscordNotification({body, title, type, url, thumbnail}) {
return result;
}
function sendSlackNotification({body, title, type, url, thumbnail}) {
const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
const data = {
blocks: [
{
type: "section",
text: {
type: "mrkdwn",
text: `*${title}*`
}
},
{
type: "section",
text: {
type: "plain_text",
text: body
}
}
]
}
// add thumbnail if exists
if (thumbnail) {
data['blocks'].push({
type: "image",
image_url: thumbnail,
alt_text: "notification_thumbnail"
});
}
data['blocks'].push(
{
type: "section",
text: {
type: "mrkdwn",
text: `<${url}|${url}>`
}
},
{
type: "context",
elements: [
{
type: "mrkdwn",
text: `*ID:* ${type}`
}
]
}
);
fetch(slack_webhook_url, {
method: 'POST',
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data),
});
}
function sendGenericNotification(data) {
const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
logger.verbose(`Sending generic notification to ${webhook_url}`);

View File

@@ -769,11 +769,6 @@
"delayed-stream": "~1.0.0"
}
},
"command-exists": {
"version": "1.2.9",
"resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz",
"integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w=="
},
"compress-commons": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.1.tgz",
@@ -2041,20 +2036,26 @@
}
},
"jsonwebtoken": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
"integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
"version": "8.5.1",
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
"integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
"requires": {
"jws": "^3.2.2",
"lodash": "^4.17.21",
"lodash.includes": "^4.3.0",
"lodash.isboolean": "^3.0.3",
"lodash.isinteger": "^4.0.4",
"lodash.isnumber": "^3.0.3",
"lodash.isplainobject": "^4.0.6",
"lodash.isstring": "^4.0.1",
"lodash.once": "^4.0.0",
"ms": "^2.1.1",
"semver": "^7.3.8"
"semver": "^5.6.0"
},
"dependencies": {
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
@@ -2199,11 +2200,41 @@
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
},
"lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8="
},
"lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
"integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY="
},
"lodash.isinteger": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
"integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M="
},
"lodash.isnumber": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
"integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w="
},
"lodash.isplainobject": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
"integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
},
"lodash.isstring": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
"integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
},
"lodash.once": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
"integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
},
"lodash.union": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz",
@@ -2602,21 +2633,11 @@
}
},
"node-id3": {
"version": "0.2.6",
"resolved": "https://registry.npmjs.org/node-id3/-/node-id3-0.2.6.tgz",
"integrity": "sha512-w8GuKXLlPpDjTxLowCt/uYMhRQzED3cg2GdSG1i6RSGKeDzPvxlXeLQuQInKljahPZ0aDnmyX7FX8BbJOM7REg==",
"version": "0.1.16",
"resolved": "https://registry.npmjs.org/node-id3/-/node-id3-0.1.16.tgz",
"integrity": "sha512-neWBJZxwrWnnebqy0b6gOGpnOPu1l1ASlusVCJUlrgr55ksftcz3lPbP/h4KaFXN+WQX7hh+kmNwkj5DMAa7KA==",
"requires": {
"iconv-lite": "0.6.2"
},
"dependencies": {
"iconv-lite": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
"integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
}
}
"iconv-lite": "^0.4.15"
}
},
"node-schedule": {
@@ -2841,13 +2862,12 @@
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
},
"passport": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz",
"integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==",
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
"integrity": "sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==",
"requires": {
"passport-strategy": "1.x.x",
"pause": "0.0.1",
"utils-merge": "^1.0.1"
"pause": "0.0.1"
}
},
"passport-http": {
@@ -3268,12 +3288,9 @@
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"semver": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
"integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
"requires": {
"lru-cache": "^6.0.0"
}
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
},
"send": {
"version": "0.18.0",

View File

@@ -17,10 +17,6 @@
"bugs": {
"url": ""
},
"engines": {
"node": "^16",
"npm": "6.14.4"
},
"homepage": "",
"dependencies": {
"@discordjs/builders": "^1.6.1",
@@ -30,7 +26,6 @@
"async-mutex": "^0.4.0",
"axios": "^0.21.2",
"bcryptjs": "^2.4.0",
"command-exists": "^1.2.9",
"compression": "^1.7.4",
"config": "^3.2.3",
"express": "^4.18.2",
@@ -39,7 +34,7 @@
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"gotify": "^1.1.0",
"jsonwebtoken": "^9.0.0",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
@@ -48,10 +43,10 @@
"mongodb": "^3.6.9",
"multer": "1.4.5-lts.1",
"node-fetch": "^2.6.7",
"node-id3": "^0.2.6",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"node-telegram-bot-api": "^0.61.0",
"passport": "^0.6.0",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.1",
"passport-ldapauth": "^3.0.1",

View File

@@ -6,21 +6,20 @@ const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const debugMode = process.env.YTDL_MODE === 'debug';
const db_api = require('./db');
const downloader_api = require('./downloader');
exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
async function subscribe(sub, user_uid = null) {
const result_obj = {
success: false,
error: ''
};
return new Promise(async resolve => {
// sub should just have url and name. here we will get isPlaylist and path
sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
@@ -33,13 +32,12 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
}
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', JSON.parse(JSON.stringify(sub)));
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = skip_get_info ? true : await getSubscriptionInfo(sub);
exports.writeSubscriptionMetadata(sub);
let success = await getSubscriptionInfo(sub);
if (success) {
if (!sub.paused) exports.getVideosForSub(sub, user_uid);
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
}
@@ -111,7 +109,7 @@ async function getSubscriptionInfo(sub) {
});
}
exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
async function unsubscribe(sub, deleteMode, user_uid = null) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -150,7 +148,7 @@ exports.unsubscribe = async (sub, deleteMode, user_uid = null) => {
await db_api.removeAllRecords('archives', {sub_id: sub.id});
}
exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
if (typeof sub === 'string') {
// TODO: fix bad workaround where sub is a sub_id
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
@@ -201,13 +199,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
return false;
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
if (deleteForever) {
// ensure video is in the archives
const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
if (!exists_in_archive) {
await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
}
} else {
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
}
return true;
@@ -218,8 +211,8 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
}
}
exports.getVideosForSub = async (sub, user_uid = null) => {
const latest_sub_obj = await exports.getSubscription(sub.id);
async function getVideosForSub(sub, user_uid = null) {
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -307,7 +300,7 @@ async function handleOutputJSON(output, sub, user_uid) {
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
@@ -318,7 +311,7 @@ async function handleOutputJSON(output, sub, user_uid) {
return files_to_download;
}
exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -371,11 +364,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...qualityPath)
// skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
const archive_count = archive_text.split('\n').length - 1;
if (archive_count > 0) {
logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
// if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`)
const archive_path = path.join(appendedBasePath, 'archive.txt');
await fs.writeFile(archive_path, archive_text);
downloadConfig.push('--download-archive', archive_path);
@@ -435,8 +428,11 @@ async function getFilesToDownload(sub, output_jsons) {
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
}
files_to_download.push(output_json);
}
@@ -445,36 +441,35 @@ async function getFilesToDownload(sub, output_jsons) {
}
exports.getSubscriptions = async (user_uid = null) => {
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
exports.getAllSubscriptions = async () => {
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}
exports.getSubscription = async (subID) => {
async function getSubscription(subID) {
// stringify and parse because we may override the 'downloading' property
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
// now with the download_queue, we may need to override 'downloading'
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
return sub;
}
exports.getSubscriptionByName = async (subName, user_uid = null) => {
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
exports.updateSubscription = async (sub) => {
async function updateSubscription(sub) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
exports.writeSubscriptionMetadata(sub);
return true;
}
exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj);
});
@@ -486,16 +481,6 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
return true;
}
exports.writeSubscriptionMetadata = (sub) => {
let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
fs.ensureDirSync(appendedBasePath);
fs.writeJSONSync(metadata_path, sub);
}
async function setFreshUploads(sub) {
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
if (!sub_files) return;
@@ -550,3 +535,17 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
getSubscriptions : getSubscriptions,
getAllSubscriptions : getAllSubscriptions,
updateSubscription : updateSubscription,
subscribe : subscribe,
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
}

View File

@@ -2,18 +2,10 @@ const db_api = require('./db');
const notifications_api = require('./notifications');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
const subscriptions_api = require('./subscriptions');
const config_api = require('./config');
const auth_api = require('./authentication/auth');
const utils = require('./utils');
const logger = require('./logger');
const CONSTS = require('./consts');
const fs = require('fs-extra');
const path = require('path');
const logger = require('./logger');
const scheduler = require('node-schedule');
const { uuid } = require('uuidv4');
const TASKS = {
backup_local_db: {
@@ -28,7 +20,7 @@ const TASKS = {
job: null
},
missing_db_records: {
run: files_api.importUnregisteredFiles,
run: db_api.importUnregisteredFiles,
title: 'Import missing DB records',
job: null
},
@@ -54,11 +46,6 @@ const TASKS = {
run: archive_api.importArchives,
title: 'Import legacy archives',
job: null
},
rebuild_database: {
run: rebuildDB,
title: 'Rebuild database',
job: null
}
}
@@ -272,73 +259,9 @@ async function autoDeleteFiles(data) {
logger.info(`Removing ${data['files_to_remove'].length} old files!`);
for (let i = 0; i < data['files_to_remove'].length; i++) {
const file_to_remove = data['files_to_remove'][i];
await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
}
}
}
async function rebuildDB() {
await db_api.backupDB();
let subs_to_add = await guessSubscriptions(false);
subs_to_add = subs_to_add.concat(await guessSubscriptions(true));
const users_to_add = await guessUsers();
for (const user_to_add of users_to_add) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_exists = await db_api.getRecord('users', {uid: user_to_add});
if (!user_exists) {
await auth_api.registerUser(user_to_add, user_to_add, 'password');
logger.info(`Regenerated user ${user_to_add}`);
}
const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add);
const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add);
subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs);
}
for (const sub_to_add of subs_to_add) {
const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid']));
// TODO: we shouldn't be creating this here
const new_sub = Object.assign({}, sub_to_add, {paused: true});
if (!sub_exists) {
await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true);
logger.info(`Regenerated subscription ${sub_to_add['name']}`);
}
}
logger.info(`Importing unregistered files`);
await files_api.importUnregisteredFiles();
}
const guessUsers = async () => {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder);
return userPaths.map(userPath => path.basename(userPath));
}
const guessSubscriptions = async (isPlaylist, basePath = null) => {
const guessed_subs = [];
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder;
const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels');
const subs = await utils.getDirectoriesInDirectory(subsPath);
for (const subPath of subs) {
const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
if (!fs.existsSync(sub_backup_path)) continue;
try {
const sub_backup = fs.readJSONSync(sub_backup_path)
delete sub_backup['_id'];
guessed_subs.push(sub_backup);
} catch(err) {
logger.warn(`Failed to reimport subscription in path ${subPath}`)
logger.warn(err);
}
}
return guessed_subs;
}
exports.TASKS = TASKS;

View File

@@ -40,7 +40,6 @@ const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const archive_api = require('../archive');
const categories_api = require('../categories');
const files_api = require('../files');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
@@ -337,22 +336,16 @@ describe('Database', async function() {
});
describe('Multi User', async function() {
const user_to_test = 'test_user';
const user_password = 'test_pass';
const sub_to_test = '';
const playlist_to_test = '';
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
await auth_api.deleteUser(user_to_test);
user = await auth_api.login('admin', 'pass');
});
describe('Basic', function() {
it('Register', async function() {
const user = await auth_api.registerUser(user_to_test, user_to_test, user_password);
assert(user);
});
it('Login', async function() {
await auth_api.registerUser(user_to_test, user_to_test, user_password);
const user = await auth_api.login(user_to_test, user_password);
describe('Authentication', function() {
it('login', async function() {
assert(user);
});
});
@@ -363,30 +356,30 @@ describe('Multi User', async function() {
});
const video_to_test = sample_video_json['uid'];
it('Get video', async function() {
const video_obj = await files_api.getVideo(video_to_test);
const video_obj = await db_api.getVideo(video_to_test);
assert(video_obj);
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
const playlist_files_to_download = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const playlist_file = await files_api.getVideo(uid, user_to_test);
const playlist_file = await db_api.getVideo(uid, user_to_test);
playlist_files_to_download.push(playlist_file);
}
const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
@@ -414,7 +407,7 @@ describe('Multi User', async function() {
// const sub_to_test = '';
// const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
// it('Get video', async function() {
// const video_obj = files_api.getVideo(video_to_test, 'admin', );
// const video_obj = db_api.getVideo(video_to_test, 'admin', );
// assert(video_obj);
// });

View File

@@ -6,7 +6,6 @@ const fs = require('fs-extra')
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
const commandExistsSync = require('command-exists').sync;
async function getCommentsForVOD(vodId) {
const exec = promisify(child_process.exec);
@@ -21,12 +20,12 @@ async function getCommentsForVOD(vodId) {
const cliExt = is_windows ? '.exe' : ''
const cliPath = `TwitchDownloaderCLI${cliExt}`
if (!commandExistsSync(cliPath)) {
if (!fs.existsSync(cliPath)) {
logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
return null;
}
const result = await exec(`${cliPath} chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);

View File

@@ -519,17 +519,6 @@ exports.convertFlatObjectToNestedObject = (obj) => {
return result;
}
exports.getDirectoriesInDirectory = async (basePath) => {
try {
const files = await fs.readdir(basePath, { withFileTypes: true });
return files
.filter((file) => file.isDirectory())
.map((file) => path.join(basePath, file.name));
} catch (err) {
return [];
}
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {

View File

@@ -21,4 +21,4 @@ version: 0.1.0
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "4.3.2"
appVersion: "4.3.1"

7
chart/LICENSE Normal file
View File

@@ -0,0 +1,7 @@
Copyright 2018 Isaac Grynsztein
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

3
chart/README.md Normal file
View File

@@ -0,0 +1,3 @@
The official YoutubeDL-Material chart
Repo: https://github.com/Tzahi12345/YoutubeDL-Material

View File

@@ -0,0 +1,4 @@
# Artifact Hub repository metadata file
repositoryID: ff79ae03-57c1-4f18-8368-5e085d06e2f1
owners:
- name: tzahi12345

22
chart/index.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: v1
entries:
youtubedl-material:
- apiVersion: v1
created: "2021-01-01T05:51:36.304331-05:00"
description: A Material Design frontend for youtube-dl
digest: f7340d24fb051ade30b890db3b5c483a884b8459316dcf2ffc6fb9413d41252e
home: https://github.com/Tzahi12345/YoutubeDL-Material/
icon: https://i.imgur.com/IKOlr0N.png
keywords:
- youtubedl-material
- youtube-dl
maintainers:
- email: IsaacMGrynsztein@gmail.com
name: tzahi12345
name: youtubedl-material
sources:
- https://github.com/Tzahi12345/YoutubeDL-Material/
urls:
- youtubedl-material-0.0.1.tgz
version: 0.0.1
generated: "2021-01-01T05:51:36.3003569-05:00"

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: ytdl-material-claim-appdata
name: ytdl-material-claim-appdata
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ default "1Gi" .Values.appdataClaimSize }}
status: {}

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: ytdl-material-claim-audio
name: ytdl-material-claim-audio
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ default "50Gi" .Values.audioClaimSize }}
status: {}

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: ytdl-material-claim-subscriptions
name: ytdl-material-claim-subscriptions
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ default "50Gi" .Values.subscriptionsClaimSize }}
status: {}

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: ytdl-material-claim-users
name: ytdl-material-claim-users
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ default "50Gi" .Values.usersClaimSize }}
status: {}

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: ytdl-material-claim-video
name: ytdl-material-claim-video
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ default "50Gi" .Values.videoClaimSize }}
status: {}

Binary file not shown.

View File

@@ -18,7 +18,6 @@ services:
- "8998:17442"
image: tzahi12345/youtubedl-material:latest
ytdl-mongo-db:
# If you are using a Raspberry Pi, use mongo:4.4.18
image: mongo:4
logging:
driver: "none"

View File

@@ -3,26 +3,13 @@ import requests
import shutil
import os
import re
import sys
from collections import OrderedDict
from github import Github
machine = platform.machine()
# https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
MACHINES_TO_ZIP = OrderedDict([
("x86_64", "Linux-x64"),
("aarch64", "LinuxArm64"),
("armv8", "LinuxArm64"),
("arm", "LinuxArm"),
("AMD64", "Windows-x64")
])
def getZipName():
for possibleMachine, possibleZipName in MACHINES_TO_ZIP.items():
if possibleMachine in machine:
return possibleZipName
def isARM():
return True if machine.startswith('arm') else False
def getLatestFileInRepo(repo, search_string):
# Create an unauthenticated instance of the Github object
@@ -59,11 +46,8 @@ def getLatestFileInRepo(repo, search_string):
print(f'No release found with {search_string}')
def getLatestCLIRelease():
zipName = getZipName()
if not zipName:
print(f"GetTwitchDownloader.py could not get valid path for '{machine}'. Exiting...")
sys.exit(1)
searchString = r'.*CLI.*' + zipName
isArm = isARM()
searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
getLatestFileInRepo("lay295/TwitchDownloader", searchString)
getLatestCLIRelease()

View File

@@ -1,39 +0,0 @@
#!/bin/sh
# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out
# and also optimizing some code with this commit.
# xoxo :D
case $(uname -m) in
x86_64)
ARCH=Linux-x64;;
aarch64)
ARCH=LinuxArm64;;
armhf)
ARCH=LinuxArm;;
armv7)
ARCH=LinuxArm;;
armv7l)
ARCH=LinuxArm;;
*)
echo "Unsupported architecture: $(uname -m)"
exit 1
esac
echo "(INFO) Architecture detected: $ARCH"
echo "(1/5) READY - Install unzip"
apt-get update && apt-get -y install unzip curl jq libicu70
VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases" | jq -r --arg arch "$ARCH" '[.[] | select(.assets | length > 0) | select(.assets[].name | contains("CLI") and contains($arch))] | max_by(.published_at) | .tag_name')
echo "(2/5) DOWNLOAD - Acquire twitchdownloader"
curl -o twitchdownloader.zip \
--connect-timeout 5 \
--max-time 120 \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \
-L "https://github.com/lay295/TwitchDownloader/releases/download/$VERSION/TwitchDownloaderCLI-$VERSION-$ARCH.zip"
unzip twitchdownloader.zip
chmod +x TwitchDownloaderCLI
echo "(3/5) Smoke test"
./TwitchDownloaderCLI --help
cp ./TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI

View File

@@ -30,7 +30,7 @@ curl -o ffmpeg.txz \
--retry 5 \
--retry-delay 0 \
--retry-max-time 40 \
"https://johnvansickle.com/ffmpeg/old-releases/ffmpeg-5.1.1-${ARCH}-static.tar.xz"
"https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
mkdir /tmp/ffmpeg
tar xf ffmpeg.txz -C /tmp/ffmpeg
echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"

821
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "youtube-dl-material",
"version": "4.3.2",
"version": "4.3.1",
"license": "MIT",
"scripts": {
"ng": "ng",
@@ -34,17 +34,18 @@
"@angular/platform-browser-dynamic": "^15.0.1",
"@angular/router": "^15.0.1",
"@fontsource/material-icons": "^4.5.4",
"@ngneat/content-loader": "^7.0.0",
"@ngneat/content-loader": "^5.0.0",
"@videogular/ngx-videogular": "^6.0.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
"file-saver": "^2.0.2",
"filesize": "^10.0.7",
"fingerprintjs2": "^2.1.0",
"fs-extra": "^10.0.0",
"material-icons": "^1.10.8",
"nan": "^2.14.1",
"ngx-avatars": "^1.4.1",
"ngx-file-drop": "^15.0.0",
"ngx-file-drop": "^13.0.0",
"rxjs": "^6.6.3",
"rxjs-compat": "^6.6.7",
"tslib": "^2.0.0",
@@ -65,6 +66,7 @@
"@typescript-eslint/parser": "^4.29.0",
"ajv": "^7.2.4",
"codelyzer": "^6.0.0",
"electron": "^19.1.9",
"eslint": "^7.32.0",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",

View File

@@ -104,7 +104,6 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
export type { SuccessObject } from './models/SuccessObject';
export type { TableInfo } from './models/TableInfo';
export type { Task } from './models/Task';
export { TaskType } from './models/TaskType';
export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
export type { TransferDBRequest } from './models/TransferDBRequest';

View File

@@ -5,4 +5,4 @@
export type AddFileToPlaylistRequest = {
file_uid: string;
playlist_id: string;
};
};

View File

@@ -13,4 +13,4 @@ export type Archive = {
sub_id?: string;
timestamp: number;
uid: string;
};
};

View File

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
export type BaseChangePermissionsRequest = {
permission: UserPermission;
new_value: YesNo;
};
};

View File

@@ -12,4 +12,4 @@ export type Category = {
* Overrides file output for downloaded files in category
*/
custom_output?: string;
};
};

View File

@@ -22,4 +22,4 @@ export namespace CategoryRule {
}
}
}

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
role: string;
});
});

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
user_uid: string;
});
});

View File

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
* UID of the concurrent stream
*/
uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
export type CheckConcurrentStreamResponse = {
stream: ConcurrentStream;
};
};

View File

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
clear_finished?: boolean;
clear_paused?: boolean;
clear_errors?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type Config = {
YoutubeDLMaterial: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { Config } from './Config';
export type ConfigResponse = {
config_file: Config;
success: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type CreateCategoryRequest = {
name: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Category } from './Category';
export type CreateCategoryResponse = {
new_category?: Category;
success?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
playlistName: string;
uids: Array<string>;
thumbnailURL: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
export type CreatePlaylistResponse = {
new_playlist: Playlist;
success: boolean;
};
};

View File

@@ -5,4 +5,4 @@
export type CropFileSettings = {
cropFileStart: number;
cropFileEnd: number;
};
};

View File

@@ -17,4 +17,4 @@ export namespace DBBackup {
}
}
}

View File

@@ -16,4 +16,4 @@ roles?: TableInfo;
download_queue?: TableInfo;
archives?: TableInfo;
};
};
};

View File

@@ -42,4 +42,4 @@ export type DatabaseFile = {
*/
abr?: number;
favorite: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
* Number of files removed
*/
delete_count?: number;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type DeleteArchiveItemsRequest = {
archives: Array<Archive>;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteCategoryRequest = {
category_uid: string;
};
};

View File

@@ -5,4 +5,4 @@
export type DeleteMp3Mp4Request = {
uid: string;
blacklistMode?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteNotificationRequest = {
uid: string;
};
};

View File

@@ -4,4 +4,4 @@
export type DeletePlaylistRequest = {
playlist_id: string;
};
};

View File

@@ -8,4 +8,4 @@ export type DeleteSubscriptionFileRequest = {
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteUserRequest = {
uid: string;
};
};

View File

@@ -27,4 +27,4 @@ export type Download = {
sub_id?: string;
sub_name?: string;
prefetched_info?: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type DownloadArchiveRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
playlist_id?: string;
url?: string;
type?: FileType;
};
};

View File

@@ -49,4 +49,4 @@ export type DownloadRequest = {
* If using youtube-dl archive, download will ignore it
*/
ignoreArchive?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type DownloadResponse = {
download?: Download;
};
};

View File

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
export type DownloadTwitchChatByVODIDResponse = {
chat: Array<TwitchChatMessage>;
};
};

View File

@@ -4,4 +4,4 @@
export type DownloadVideosForSubscriptionRequest = {
subID: string;
};
};

View File

@@ -5,4 +5,4 @@
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}
}

View File

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
AUDIO_ONLY = 'audio_only',
VIDEO_ONLY = 'video_only',
BOTH = 'both',
}
}

View File

@@ -4,4 +4,4 @@
export type GenerateArgsResponse = {
args?: Array<string>;
};
};

View File

@@ -4,4 +4,4 @@
export type GenerateNewApiKeyResponse = {
new_api_key: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Category } from './Category';
export type GetAllCategoriesResponse = {
categories: Array<Category>;
};
};

View File

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
* Filters downloads with the array
*/
uids?: Array<string> | null;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetAllDownloadsResponse = {
downloads?: Array<Download>;
};
};

View File

@@ -21,4 +21,4 @@ export type GetAllFilesRequest = {
* Include if you want to filter by subscription
*/
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
export type GetAllSubscriptionsResponse = {
subscriptions: Array<Subscription>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Task } from './Task';
export type GetAllTasksResponse = {
tasks?: Array<Task>;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type GetArchivesRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type GetArchivesResponse = {
archives: Array<Archive>;
};
};

View File

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
export type GetDBBackupsResponse = {
tasks?: Array<DBBackup>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetDownloadRequest = {
download_uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetDownloadResponse = {
download?: Download;
};
};

View File

@@ -4,4 +4,4 @@
export type GetFileFormatsRequest = {
url?: string;
};
};

View File

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
result: {
formats?: Array<any>;
};
};
};

View File

@@ -14,4 +14,4 @@ export type GetFileRequest = {
* User UID
*/
uuid?: string;
};
};

Some files were not shown because too many files have changed in this diff Show More