Compare commits

...

16 Commits

Author SHA1 Message Date
Tzahi12345
010f0fbb1c Added ability to set a pin for settings menu 2023-04-27 21:37:32 -04:00
Tzahi12345
f973426bd2 Hotfix for error that prevents downloads from occurring 2023-04-24 21:11:10 -04:00
Tzahi12345
5a379a6a2b Updated package-lock.json (#877) 2023-04-24 20:03:11 -04:00
Tzahi12345
d76aaf83f6 Merge pull request #707 from Tzahi12345/categories-fix
Categories matching bug fix
2023-04-24 10:35:08 -04:00
Tzahi12345
d3b88412c6 Fixed thumbnails for auto-generated playlists 2023-04-23 22:27:09 -04:00
Tzahi12345
6cee892e18 Added label for category field in video-info-dialog 2023-04-23 22:20:50 -04:00
Tzahi12345
e2438a236b Adjusted category UI styling to Angular 15 updates 2023-04-23 22:14:20 -04:00
Tzahi12345
955c401f0b Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into categories-fix 2023-04-22 17:47:16 -04:00
Tzahi12345
527b1f1cb9 Merge pull request #872 from Tzahi12345/twitch-downloader-fix
Twitch chat download fixes
2023-04-21 00:16:06 -04:00
Tzahi12345
24d8072eb5 Fixed minor syntax error in Dockerfile 2023-04-20 21:47:20 -04:00
Tzahi12345
c81bf980ca Separated image for TwitchDownloader download in Dockerfile 2023-04-20 21:41:36 -04:00
Tzahi12345
a91381720f Added link in error message to get TiwtchDownloaderCLI 2023-04-20 21:16:06 -04:00
Tzahi12345
edd4a0928c Fixed twitch downloading by using TiwtchDownloader's CLI
Removed unecessary settings

Created dedicated folder for docker utils
2023-04-20 21:11:48 -04:00
Isaac Abadi
2e52ec22e0 Default sort for videos is now download date 2022-09-15 21:38:19 -04:00
Isaac Abadi
efdd0dd228 Categories fix for yt-dlp 2022-09-15 21:38:00 -04:00
Isaac Abadi
415c97cb09 Fixed issue where categories were not being properly applied to matching files (#701) 2022-07-07 01:07:22 -04:00
164 changed files with 666 additions and 228 deletions

View File

@@ -2,7 +2,7 @@
FROM ubuntu:22.04 AS ffmpeg
ENV DEBIAN_FRONTEND=noninteractive
# Use script due local build compability
COPY ffmpeg-fetch.sh .
COPY docker-utils/ffmpeg-fetch.sh .
RUN chmod +x ffmpeg-fetch.sh
RUN sh ./ffmpeg-fetch.sh
@@ -47,6 +47,15 @@ RUN npm config set strict-ssl false && \
npm install --prod && \
ls -al
FROM base as python
WORKDIR /app
COPY docker-utils/GetTwitchDownloader.py .
RUN apt update && \
apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install PyGithub requests
RUN python GetTwitchDownloader.py
# Final image
FROM base
@@ -55,7 +64,7 @@ RUN npm install -g pm2 && \
apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
apt clean && \
rm -rf /var/lib/apt/lists/*
RUN pip install tdh-tcd pycryptodomex
RUN pip install pycryptodomex
WORKDIR /app
# User 1000 already exist from base image
COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]

View File

@@ -678,22 +678,6 @@ paths:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/isPinSet:
post:
tags:
- security
summary: Check if pin is set
description: Checks if the pin is set for settings
operationId: post-api-isPinSet
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/inline_response_200_15'
security:
- Auth query parameter: []
/api/generateNewAPIKey:
post:
tags:
@@ -1311,6 +1295,48 @@ paths:
- Auth query parameter: []
tags:
- multi-user mode
/api/setPin:
post:
summary: Set settings pin
operationId: post-api-setPin
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/SetPinRequest'
description: 'Sets a pin for the settings'
security:
- Auth query parameter: []
tags:
- security
/api/auth/pinLogin:
post:
summary: Pin login
operationId: post-api-pin-login
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/PinLoginResponse'
description: Use this endpoint to generate a JWT token for pin authentication. Put anything in the username field.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/LoginRequest'
security:
- Auth query parameter: []
tags:
- security
/api/getUsers:
post:
summary: Get all users
@@ -3025,6 +3051,13 @@ components:
type: string
required:
- role
SetPinRequest:
required:
- new_pin
type: object
properties:
new_pin:
type: string
file:
title: file
type: object
@@ -3074,6 +3107,13 @@ components:
type: array
items:
$ref: '#/components/schemas/UserPermission'
PinLoginResponse:
required:
- pin_token
type: object
properties:
pin_token:
type: string
UpdateUserRequest:
required:
- change_object

View File

@@ -162,6 +162,7 @@ app.use(bodyParser.json());
// use passport
app.use(auth_api.passport.initialize());
app.use(auth_api.passport.session());
// actual functions
@@ -741,6 +742,18 @@ const optionalJwt = async function (req, res, next) {
return next();
};
const optionalPin = async function (req, res, next) {
const use_pin = config_api.getConfigItem('ytdl_use_pin');
if (use_pin && req.path.includes('/api/setConfig')) {
if (!req.query.pin_token) {
res.sendStatus(418); // I'm a teapot (RFC 2324)
return;
}
return next();
}
return next();
};
app.get('/api/config', function(req, res) {
let config_file = config_api.getConfigFile();
res.send({
@@ -749,7 +762,7 @@ app.get('/api/config', function(req, res) {
});
});
app.post('/api/setConfig', optionalJwt, function(req, res) {
app.post('/api/setConfig', optionalJwt, optionalPin, function(req, res) {
let new_config_file = req.body.new_config_file;
if (new_config_file && new_config_file['YoutubeDLMaterial']) {
let success = config_api.setConfigFile(new_config_file);
@@ -1933,12 +1946,23 @@ app.post('/api/auth/login'
, auth_api.generateJWT
, auth_api.returnAuthResponse
);
app.post('/api/auth/pinLogin'
, auth_api.passport.authenticate(['local_pin'], {})
, auth_api.generatePinJWT
, auth_api.returnPinAuthResponse
);
app.post('/api/auth/jwtAuth'
, auth_api.passport.authenticate('jwt', { session: false })
, auth_api.passport.authorize('jwt')
, auth_api.generateJWT
, auth_api.returnAuthResponse
);
app.post('/api/auth/pinAuth'
, auth_api.passport.authenticate('pin', { session: false })
, auth_api.passport.authorize('pin')
, auth_api.generatePinJWT
, auth_api.returnPinAuthResponse
);
app.post('/api/auth/changePassword', optionalJwt, async (req, res) => {
let user_uid = req.body.user_uid;
let password = req.body.new_password;
@@ -2028,6 +2052,13 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
res.send({success: success});
});
app.post('/api/setPin', function(req, res) {
const success = auth_api.setPin(req.body.new_pin);
res.send({
success: success
});
});
// notifications
app.post('/api/getNotifications', optionalJwt, async (req, res) => {

View File

@@ -15,7 +15,6 @@ var JwtStrategy = require('passport-jwt').Strategy,
// other required vars
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function () {
@@ -50,11 +49,11 @@ exports.initialize = function () {
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
opts = {}
const opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
exports.passport.use('jwt', new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
@@ -63,6 +62,21 @@ exports.initialize = function () {
// or you could create a new account
}
}));
const pin_opts = {}
pin_opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('pin_token');
pin_opts.secretOrKey = SERVER_SECRET;
exports.passport.use('pin', new JwtStrategy(pin_opts, {
passwordField: 'pin'},
async function(username, password, done) {
if (await bcrypt.compare(password, config_api.getConfigItem('ytdl_pin_hash'))) {
return done(null, { success: true });
} else {
return done(null, false, { message: 'Incorrect pin' });
}
}
));
}
const setupRoles = async () => {
@@ -188,6 +202,10 @@ exports.login = async (username, password) => {
return await bcrypt.compare(password, user.passhash) ? user : false;
}
exports.pinLogin = async (pin) => {
return await bcrypt.compare(pin, config_api.getConfigItem('ytdl_pin_hash'));
}
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
@@ -196,6 +214,14 @@ exports.passport.use(new LocalStrategy({
}
));
exports.passport.use('local_pin', new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
async function(username, password, done) {
return done(null, await exports.pinLogin(password));
}
));
var getLDAPConfiguration = function(req, callback) {
const ldap_config = config_api.getConfigItem('ytdl_ldap_config');
const opts = {server: ldap_config};
@@ -237,6 +263,14 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.generatePinJWT = function(req, res, next) {
var payload = {
exp: Math.floor(Date.now() / 1000) + JWT_EXPIRATION
};
req.token = jwt.sign(payload, SERVER_SECRET);
next();
}
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
@@ -246,6 +280,12 @@ exports.returnAuthResponse = async function(req, res) {
});
}
exports.returnPinAuthResponse = async function(req, res) {
res.status(200).json({
pin_token: req.token
});
}
/***************************************
* Authorization: middleware that checks the
* JWT token for validity before allowing
@@ -439,6 +479,13 @@ exports.userPermissions = async function(user_uid) {
return user_permissions;
}
// pin
exports.setPin = async (new_pin) => {
const pin_hash = await bcrypt.hash(new_pin, saltRounds);
return config_api.setConfigItem('ytdl_pin_hash', pin_hash);
}
function getToken(queryParams) {
if (queryParams && queryParams.jwt) {
var parted = queryParams.jwt.split(' ');
@@ -450,7 +497,7 @@ function getToken(queryParams) {
} else {
return null;
}
};
}
function generateUserObject(userid, username, hash, auth_method = 'internal') {
let new_user = {

View File

@@ -202,15 +202,14 @@ const DEFAULT_CONFIG = {
"enable_all_notifications": true,
"allowed_notification_types": [],
"enable_rss_feed": false,
"use_pin": false,
"pin_hash": "",
},
"API": {
"use_API_key": false,
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false,

View File

@@ -92,6 +92,14 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_enable_rss_feed',
'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
},
'ytdl_use_pin': {
'key': 'ytdl_use_pin',
'path': 'YoutubeDLMaterial.Extra.use_pin'
},
'ytdl_pin_hash': {
'key': 'ytdl_pin_hash',
'path': 'YoutubeDLMaterial.Extra.pin_hash'
},
// API
'ytdl_use_api_key': {
@@ -110,18 +118,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_youtube_api_key',
'path': 'YoutubeDLMaterial.API.youtube_API_key'
},
'ytdl_use_twitch_api': {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_client_id': {
'key': 'ytdl_twitch_client_id',
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
},
'ytdl_twitch_client_secret': {
'key': 'ytdl_twitch_client_secret',
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'

View File

@@ -245,11 +245,10 @@ async function collectInfo(download_uid) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await exports.generateArgs(url, type, options, download['user_uid']);
args = utils.filterArgs(args, ['--no-simulate']);
info = await exports.getVideoInfoByURL(url, args, download_uid);
}
download['category'] = category;
const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
// setup info required to calculate download progress
@@ -272,6 +271,7 @@ async function collectInfo(download_uid) {
files_to_check_for_progress: files_to_check_for_progress,
expected_file_size: expected_file_size,
title: playlist_title ? playlist_title : info['title'],
category: stripped_category,
prefetched_info: null
});
}
@@ -350,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -552,7 +552,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const temp_args = utils.filterArgs(args, ['--no-simulate']);
const new_args = [...temp_args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {

View File

@@ -255,7 +255,7 @@
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
},
"array.prototype.findindex": {
"version": "2.2.1",
@@ -891,7 +891,7 @@
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
"integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
},
"enabled": {
"version": "2.0.0",
@@ -901,7 +901,7 @@
"encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
},
"end-of-stream": {
"version": "1.4.4",
@@ -1012,7 +1012,7 @@
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
},
"escape-string-regexp": {
"version": "4.0.0",
@@ -1027,7 +1027,7 @@
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="
},
"eventemitter3": {
"version": "3.1.2",
@@ -1122,6 +1122,33 @@
}
}
},
"express-session": {
"version": "1.17.3",
"resolved": "https://registry.npmjs.org/express-session/-/express-session-1.17.3.tgz",
"integrity": "sha512-4+otWXlShYlG1Ma+2Jnn+xgKUZTMJ5QD3YvfilX3AcocOAbIkVylSWEklzALe/+Pu4qV6TYBj5GwOBFfdKqLBw==",
"requires": {
"cookie": "0.4.2",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "~2.0.0",
"on-headers": "~1.0.2",
"parseurl": "~1.3.3",
"safe-buffer": "5.2.1",
"uid-safe": "~2.1.5"
},
"dependencies": {
"depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
},
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
}
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -1256,7 +1283,7 @@
"fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
"integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q=="
},
"fs-constants": {
"version": "1.0.0",
@@ -1521,9 +1548,9 @@
}
},
"http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
"integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
},
"http-errors": {
"version": "1.8.1",
@@ -2167,7 +2194,7 @@
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
"integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
},
"merge-stream": {
"version": "2.0.0",
@@ -2177,7 +2204,7 @@
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
"integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="
},
"mime": {
"version": "1.6.0",
@@ -2769,12 +2796,12 @@
"path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"pause": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz",
"integrity": "sha1-HUCLP9t2kjuVQ9lvtMnf1TXZy10="
"integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg=="
},
"performance-now": {
"version": "2.1.0",
@@ -2847,6 +2874,11 @@
"resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz",
"integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw=="
},
"random-bytes": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz",
"integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ=="
},
"randombytes": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -2905,13 +2937,24 @@
}
},
"regexp.prototype.flags": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",
"integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==",
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz",
"integrity": "sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.3",
"functions-have-names": "^1.2.2"
"define-properties": "^1.2.0",
"functions-have-names": "^1.2.3"
},
"dependencies": {
"define-properties": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz",
"integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==",
"requires": {
"has-property-descriptors": "^1.0.0",
"object-keys": "^1.1.1"
}
}
}
},
"request": {
@@ -3455,6 +3498,14 @@
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"uid-safe": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
"integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==",
"requires": {
"random-bytes": "~1.0.0"
}
},
"unbox-primitive": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz",
@@ -3474,7 +3525,7 @@
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
"integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
},
"unzipper": {
"version": "0.10.10",

View File

@@ -27,6 +27,7 @@
"compression": "^1.7.4",
"config": "^3.2.3",
"express": "^4.17.3",
"express-session": "^1.17.3",
"feed": "^4.2.2",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",

View File

@@ -508,7 +508,7 @@ describe('Downloader', function() {
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1493770675';
const example_vod = '1710641401';
it('Download VOD', async function() {
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);

View File

@@ -4,19 +4,28 @@ const logger = require('./logger');
const moment = require('moment');
const fs = require('fs-extra')
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
async function getCommentsForVOD(clientID, clientSecret, vodId) {
const { promisify } = require('util');
const child_process = require('child_process');
async function getCommentsForVOD(vodId) {
const exec = promisify(child_process.exec);
// Reject invalid params to prevent command injection attack
if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
if (!vodId.match(/^[0-9a-z]+$/)) {
logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
return null;
}
const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
const is_windows = process.platform === 'win32';
const cliExt = is_windows ? '.exe' : ''
const cliPath = `TwitchDownloaderCLI${cliExt}`
if (!fs.existsSync(cliPath)) {
logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
return null;
}
const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -73,9 +82,7 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
const chat = await getCommentsForVOD(vodId);
// save file if needed params are included
let file_path = null;

View File

@@ -0,0 +1,53 @@
import platform
import requests
import shutil
import os
import re
from github import Github
machine = platform.machine()
def isARM():
return True if machine.startswith('arm') else False
def getLatestFileInRepo(repo, search_string):
# Create an unauthenticated instance of the Github object
g = Github(os.environ.get('GH_TOKEN'))
# Replace with the repository owner and name
repo = g.get_repo(repo)
# Get all releases of the repository
releases = repo.get_releases()
# Loop through the releases in reverse order (from latest to oldest)
for release in list(releases):
# Get the release assets (files attached to the release)
assets = release.get_assets()
# Loop through the assets
for asset in assets:
if re.search(search_string, asset.name):
print(f'Downloading: {asset.name}')
response = requests.get(asset.browser_download_url)
with open(asset.name, 'wb') as f:
f.write(response.content)
print(f'Download complete: {asset.name}. Unzipping...')
shutil.unpack_archive(asset.name, './')
print(f'Unzipping complete!')
os.remove(asset.name)
break
else:
continue
break
else:
# If no matching release is found, print a message
print(f'No release found with {search_string}')
def getLatestCLIRelease():
isArm = isARM()
searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
getLatestFileInRepo("lay295/TwitchDownloader", searchString)
getLatestCLIRelease()

2
package-lock.json generated
View File

@@ -1,6 +1,6 @@
{
"name": "youtube-dl-material",
"version": "4.3.0",
"version": "4.3.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View File

@@ -87,6 +87,7 @@ export type { LoginResponse } from './models/LoginResponse';
export type { Notification } from './models/Notification';
export { NotificationAction } from './models/NotificationAction';
export { NotificationType } from './models/NotificationType';
export type { PinLoginResponse } from './models/PinLoginResponse';
export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse';
@@ -95,6 +96,7 @@ export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
export { Schedule } from './models/Schedule';
export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SetNotificationsToReadRequest } from './models/SetNotificationsToReadRequest';
export type { SetPinRequest } from './models/SetPinRequest';
export type { SharingToggle } from './models/SharingToggle';
export type { Sort } from './models/Sort';
export type { SubscribeRequest } from './models/SubscribeRequest';

View File

@@ -5,4 +5,4 @@
export type AddFileToPlaylistRequest = {
file_uid: string;
playlist_id: string;
};
};

View File

@@ -13,4 +13,4 @@ export type Archive = {
sub_id?: string;
timestamp: number;
uid: string;
};
};

View File

@@ -8,4 +8,4 @@ import type { YesNo } from './YesNo';
export type BaseChangePermissionsRequest = {
permission: UserPermission;
new_value: YesNo;
};
};

View File

@@ -12,4 +12,4 @@ export type Category = {
* Overrides file output for downloaded files in category
*/
custom_output?: string;
};
};

View File

@@ -22,4 +22,4 @@ export namespace CategoryRule {
}
}
}

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeRolePermissionsRequest = (BaseChangePermissionsRequest & {
role: string;
});
});

View File

@@ -6,4 +6,4 @@ import type { BaseChangePermissionsRequest } from './BaseChangePermissionsReques
export type ChangeUserPermissionsRequest = (BaseChangePermissionsRequest & {
user_uid: string;
});
});

View File

@@ -7,4 +7,4 @@ export type CheckConcurrentStreamRequest = {
* UID of the concurrent stream
*/
uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { ConcurrentStream } from './ConcurrentStream';
export type CheckConcurrentStreamResponse = {
stream: ConcurrentStream;
};
};

View File

@@ -6,4 +6,4 @@ export type ClearDownloadsRequest = {
clear_finished?: boolean;
clear_paused?: boolean;
clear_errors?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type ConcurrentStream = {
playback_timestamp?: number;
unix_timestamp?: number;
playing?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type Config = {
YoutubeDLMaterial: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { Config } from './Config';
export type ConfigResponse = {
config_file: Config;
success: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type CreateCategoryRequest = {
name: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Category } from './Category';
export type CreateCategoryResponse = {
new_category?: Category;
success?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type CreatePlaylistRequest = {
playlistName: string;
uids: Array<string>;
thumbnailURL: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Playlist } from './Playlist';
export type CreatePlaylistResponse = {
new_playlist: Playlist;
success: boolean;
};
};

View File

@@ -5,4 +5,4 @@
export type CropFileSettings = {
cropFileStart: number;
cropFileEnd: number;
};
};

View File

@@ -17,4 +17,4 @@ export namespace DBBackup {
}
}
}

View File

@@ -16,4 +16,4 @@ roles?: TableInfo;
download_queue?: TableInfo;
archives?: TableInfo;
};
};
};

View File

@@ -42,4 +42,4 @@ export type DatabaseFile = {
*/
abr?: number;
favorite: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type DeleteAllFilesResponse = {
* Number of files removed
*/
delete_count?: number;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type DeleteArchiveItemsRequest = {
archives: Array<Archive>;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteCategoryRequest = {
category_uid: string;
};
};

View File

@@ -5,4 +5,4 @@
export type DeleteMp3Mp4Request = {
uid: string;
blacklistMode?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteNotificationRequest = {
uid: string;
};
};

View File

@@ -4,4 +4,4 @@
export type DeletePlaylistRequest = {
playlist_id: string;
};
};

View File

@@ -8,4 +8,4 @@ export type DeleteSubscriptionFileRequest = {
* If true, does not remove id from archive. Only valid if youtube-dl archive is enabled in settings.
*/
deleteForever?: boolean;
};
};

View File

@@ -4,4 +4,4 @@
export type DeleteUserRequest = {
uid: string;
};
};

View File

@@ -27,4 +27,4 @@ export type Download = {
sub_id?: string;
sub_name?: string;
prefetched_info?: any;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type DownloadArchiveRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type DownloadFileRequest = {
playlist_id?: string;
url?: string;
type?: FileType;
};
};

View File

@@ -49,4 +49,4 @@ export type DownloadRequest = {
* If using youtube-dl archive, download will ignore it
*/
ignoreArchive?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type DownloadResponse = {
download?: Download;
};
};

View File

@@ -20,4 +20,4 @@ export type DownloadTwitchChatByVODIDRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -6,4 +6,4 @@ import type { TwitchChatMessage } from './TwitchChatMessage';
export type DownloadTwitchChatByVODIDResponse = {
chat: Array<TwitchChatMessage>;
};
};

View File

@@ -4,4 +4,4 @@
export type DownloadVideosForSubscriptionRequest = {
subID: string;
};
};

View File

@@ -5,4 +5,4 @@
export enum FileType {
AUDIO = 'audio',
VIDEO = 'video',
}
}

View File

@@ -6,4 +6,4 @@ export enum FileTypeFilter {
AUDIO_ONLY = 'audio_only',
VIDEO_ONLY = 'video_only',
BOTH = 'both',
}
}

View File

@@ -4,4 +4,4 @@
export type GenerateArgsResponse = {
args?: Array<string>;
};
};

View File

@@ -4,4 +4,4 @@
export type GenerateNewApiKeyResponse = {
new_api_key: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Category } from './Category';
export type GetAllCategoriesResponse = {
categories: Array<Category>;
};
};

View File

@@ -7,4 +7,4 @@ export type GetAllDownloadsRequest = {
* Filters downloads with the array
*/
uids?: Array<string> | null;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetAllDownloadsResponse = {
downloads?: Array<Download>;
};
};

View File

@@ -21,4 +21,4 @@ export type GetAllFilesRequest = {
* Include if you want to filter by subscription
*/
sub_id?: string;
};
};

View File

@@ -11,4 +11,4 @@ export type GetAllFilesResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Subscription } from './Subscription';
export type GetAllSubscriptionsResponse = {
subscriptions: Array<Subscription>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Task } from './Task';
export type GetAllTasksResponse = {
tasks?: Array<Task>;
};
};

View File

@@ -7,4 +7,4 @@ import type { FileType } from './FileType';
export type GetArchivesRequest = {
type?: FileType;
sub_id?: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Archive } from './Archive';
export type GetArchivesResponse = {
archives: Array<Archive>;
};
};

View File

@@ -6,4 +6,4 @@ import type { DBBackup } from './DBBackup';
export type GetDBBackupsResponse = {
tasks?: Array<DBBackup>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetDownloadRequest = {
download_uid: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Download } from './Download';
export type GetDownloadResponse = {
download?: Download;
};
};

View File

@@ -4,4 +4,4 @@
export type GetFileFormatsRequest = {
url?: string;
};
};

View File

@@ -7,4 +7,4 @@ export type GetFileFormatsResponse = {
result: {
formats?: Array<any>;
};
};
};

View File

@@ -14,4 +14,4 @@ export type GetFileRequest = {
* User UID
*/
uuid?: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { DatabaseFile } from './DatabaseFile';
export type GetFileResponse = {
success: boolean;
file?: DatabaseFile;
};
};

View File

@@ -16,4 +16,4 @@ export type GetFullTwitchChatRequest = {
*/
uuid?: string;
sub?: Subscription;
};
};

View File

@@ -5,4 +5,4 @@
export type GetFullTwitchChatResponse = {
success: boolean;
error?: string;
};
};

View File

@@ -4,4 +4,4 @@
export type GetLogsRequest = {
lines?: number;
};
};

View File

@@ -8,4 +8,4 @@ export type GetLogsResponse = {
*/
logs?: string;
success?: boolean;
};
};

View File

@@ -11,4 +11,4 @@ export type GetMp3sResponse = {
* All audio playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -11,4 +11,4 @@ export type GetMp4sResponse = {
* All video playlists
*/
playlists: Array<Playlist>;
};
};

View File

@@ -6,4 +6,4 @@ import type { Notification } from './Notification';
export type GetNotificationsResponse = {
notifications?: Array<Notification>;
};
};

View File

@@ -9,4 +9,4 @@ export type GetPlaylistRequest = {
type?: FileType;
uuid?: string;
include_file_metadata?: boolean;
};
};

View File

@@ -12,4 +12,4 @@ export type GetPlaylistResponse = {
* File objects for every uid in the playlist's uids property, in the same order
*/
file_objs?: Array<DatabaseFile>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetPlaylistsRequest = {
include_categories?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ import type { Playlist } from './Playlist';
export type GetPlaylistsResponse = {
playlists: Array<Playlist>;
};
};

View File

@@ -13,4 +13,4 @@ user?: {
permissions?: Array<UserPermission>;
};
};
};
};

View File

@@ -11,4 +11,4 @@ export type GetSubscriptionRequest = {
* Subscription name
*/
name?: string;
};
};

View File

@@ -7,4 +7,4 @@ import type { Subscription } from './Subscription';
export type GetSubscriptionResponse = {
subscription: Subscription;
files: Array<any>;
};
};

View File

@@ -4,4 +4,4 @@
export type GetTaskRequest = {
task_key: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { Task } from './Task';
export type GetTaskResponse = {
task?: Task;
};
};

View File

@@ -6,4 +6,4 @@ import type { User } from './User';
export type GetUsersResponse = {
users: Array<User>;
};
};

View File

@@ -8,4 +8,4 @@ export type ImportArchiveRequest = {
archive: string;
type: FileType;
sub_id?: string;
};
};

View File

@@ -9,4 +9,4 @@ export type IncrementViewCountRequest = {
* User UID
*/
uuid?: string;
};
};

View File

@@ -5,4 +5,4 @@
export type LoginRequest = {
username: string;
password: string;
};
};

View File

@@ -10,4 +10,4 @@ export type LoginResponse = {
token?: string;
permissions?: Array<UserPermission>;
available_permissions?: Array<UserPermission>;
};
};

View File

@@ -13,4 +13,4 @@ export type Notification = {
read: boolean;
data?: any;
timestamp: number;
};
};

View File

@@ -7,4 +7,4 @@ export enum NotificationAction {
RETRY_DOWNLOAD = 'retry_download',
VIEW_DOWNLOAD_ERROR = 'view_download_error',
VIEW_TASKS = 'view_tasks',
}
}

View File

@@ -6,4 +6,4 @@ export enum NotificationType {
DOWNLOAD_COMPLETE = 'download_complete',
DOWNLOAD_ERROR = 'download_error',
TASK_FINISHED = 'task_finished',
}
}

View File

@@ -0,0 +1,7 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type PinLoginResponse = {
pin_token: string;
};

View File

@@ -15,4 +15,4 @@ export type Playlist = {
user_uid?: string;
auto?: boolean;
sharingEnabled?: boolean;
};
};

View File

@@ -6,4 +6,4 @@ export type RegisterRequest = {
userid: string;
username: string;
password: string;
};
};

View File

@@ -6,4 +6,4 @@ import type { User } from './User';
export type RegisterResponse = {
user?: User;
};
};

Some files were not shown because too many files have changed in this diff Show More