TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-29 16:10:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into categories-fix

Browse Source
This commit is contained in:
Tzahi12345
2023-04-22 17:47:16 -04:00
parent 2e52ec22e0 527b1f1cb9
commit 955c401f0b
178 changed files with 40035 additions and 4803 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
backend/app.js
View File

@@ -18,6 +18,7 @@ const URL = require('url').URL;
const CONSTS = require('./consts')
const read_last_lines = require('read-last-lines');
const ps = require('ps-node');
const Feed = require('feed').Feed;
// needed if bin/details somehow gets deleted
if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"})
@@ -32,6 +33,7 @@ const subscriptions_api = require('./subscriptions');
const categories_api = require('./categories');
const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
var app = express();
@@ -69,7 +71,8 @@ db.defaults(
downloads: {},
subscriptions: [],
files_to_db_migration_complete: false,
tasks_manager_role_migration_complete: false
tasks_manager_role_migration_complete: false,
archives_migration_complete: false
}).write();
users_db.defaults(
@@ -200,6 +203,15 @@ async function checkMigrations() {
db.set('tasks_manager_role_migration_complete', true).write();
}
const archives_migration_complete = db.get('archives_migration_complete').value();
if (!archives_migration_complete) {
logger.info('Checking if archives have been migrated...');
const imported_archives = await archive_api.importArchives();
if (imported_archives) logger.info('Archives migration complete!');
else logger.error('Failed to migrate archives!');
db.set('archives_migration_complete', true).write();
}
return true;
}
@@ -511,9 +523,6 @@ async function loadConfig() {
db_api.database_initialized = true;
db_api.database_initialized_bs.next(true);
// creates archive path if missing
await fs.ensureDir(utils.getArchiveFolder());
// check migrations
await checkMigrations();
@@ -559,14 +568,7 @@ function loadConfigValues() {
url_domain = new URL(url);
let logger_level = config_api.getConfigItem('ytdl_logger_level');
const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
if (!possible_levels.includes(logger_level)) {
logger.error(`${logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
logger_level = 'info';
}
logger.level = logger_level;
winston.loggers.get('console').level = logger_level;
logger.transports[2].level = logger_level;
utils.updateLoggerLevel(logger_level);
}
function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
@@ -701,7 +703,7 @@ app.use(function(req, res, next) {
next();
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
next();
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/')) {
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
next();
} else {
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -774,7 +776,7 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
const db_info = await db_api.getDBStats();
res.send({db_info: db_info});
res.send(db_info);
});
app.post('/api/transferDB', optionalJwt, async (req, res) => {
@@ -814,11 +816,13 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
additionalArgs: req.body.additionalArgs,
customOutput: req.body.customOutput,
selectedHeight: req.body.selectedHeight,
maxHeight: req.body.maxHeight,
customQualityConfiguration: req.body.customQualityConfiguration,
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
ui_uid: req.body.ui_uid,
cropFileSettings: req.body.cropFileSettings
cropFileSettings: req.body.cropFileSettings,
ignoreArchive: req.body.ignoreArchive
};
const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -844,6 +848,7 @@ app.post('/api/generateArgs', optionalJwt, async function(req, res) {
additionalArgs: req.body.additionalArgs,
customOutput: req.body.customOutput,
selectedHeight: req.body.selectedHeight,
maxHeight: req.body.maxHeight,
customQualityConfiguration: req.body.customQualityConfiguration,
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
@@ -922,35 +927,15 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
// these are returned
let files = null;
const sort = req.body.sort;
const range = req.body.range;
const text_search = req.body.text_search;
const file_type_filter = req.body.file_type_filter;
const favorite_filter = req.body.favorite_filter;
const sub_id = req.body.sub_id;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
const file_count = await db_api.getRecords('files', filter_obj, true);
files = JSON.parse(JSON.stringify(files));
const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
res.send({
files: files,
@@ -1093,9 +1078,6 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
} else if (is_playlist) {
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
} else if (type === 'subscription') {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
} else {
// error
success = false;
@@ -1110,7 +1092,7 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
});
});
app.post('/api/incrementViewCount', optionalJwt, async (req, res) => {
app.post('/api/incrementViewCount', async (req, res) => {
let file_uid = req.body.file_uid;
let sub_id = req.body.sub_id;
let uuid = req.body.uuid;
@@ -1245,12 +1227,9 @@ app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
let deleteForever = req.body.deleteForever;
let file = req.body.file;
let file_uid = req.body.file_uid;
let sub = req.body.sub;
let user_uid = req.isAuthenticated() ? req.user.uid : null;
let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, file_uid, user_uid);
let success = await db_api.deleteFile(file_uid, deleteForever);
if (success) {
res.send({
@@ -1431,10 +1410,9 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
app.post('/api/deleteFile', optionalJwt, async (req, res) => {
const uid = req.body.uid;
const blacklistMode = req.body.blacklistMode;
const uuid = req.isAuthenticated() ? req.user.uid : null;
let wasDeleted = false;
wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
wasDeleted = await db_api.deleteFile(uid, blacklistMode);
res.send(wasDeleted);
});
@@ -1466,7 +1444,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
for (let i = 0; i < files.length; i++) {
let wasDeleted = false;
wasDeleted = await db_api.deleteFile(files[i].uid, uuid, blacklistMode);
wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode);
if (wasDeleted) {
delete_count++;
}
@@ -1527,20 +1505,69 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
});
});
app.post('/api/getArchives', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const sub_id = req.body.sub_id;
const filter_obj = {user_uid: uuid, sub_id: sub_id};
const type = req.body.type;
// we do this for file types because if type is null, that means get files of all types
if (type) filter_obj['type'] = type;
const archives = await db_api.getRecords('archives', filter_obj);
res.send({
archives: archives
});
});
app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
let sub = req.body.sub;
let archive_dir = sub.archive;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const sub_id = req.body.sub_id;
const type = req.body.type;
let full_archive_path = path.join(archive_dir, 'archive.txt');
const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
if (await fs.pathExists(full_archive_path)) {
res.sendFile(full_archive_path);
if (archive_text !== null && archive_text !== undefined) {
res.setHeader('Content-type', "application/octet-stream");
res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
res.send(archive_text);
} else {
res.sendStatus(404);
res.sendStatus(400);
}
});
app.post('/api/importArchive', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const archive = req.body.archive;
const sub_id = req.body.sub_id;
const type = req.body.type;
const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
res.send({
success: !!imported_count,
imported_count: imported_count
});
});
app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const archives = req.body.archives;
let success = true;
for (const archive of archives) {
success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
}
res.send({
success: success
});
});
var upload_multer = multer({ dest: __dirname + '/appdata/' });
app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
@@ -1732,8 +1759,8 @@ app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
app.post('/api/restartDownload', optionalJwt, async (req, res) => {
const download_uid = req.body.download_uid;
const success = await downloader_api.restartDownload(download_uid);
res.send({success: success});
const new_download = await downloader_api.restartDownload(download_uid);
res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
});
app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
@@ -1810,6 +1837,15 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
res.send({success: success});
});
app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const new_options = req.body.new_options;
const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
res.send({success: success});
});
app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
const backup_dir = path.join('appdata', 'db_backup');
fs.ensureDirSync(backup_dir);
@@ -1993,6 +2029,93 @@ app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
res.send({success: success});
});
// notifications
app.post('/api/getNotifications', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
res.send({notifications: notifications});
});
// set notifications to read
app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
res.send({success: success});
});
app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
const uid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.removeRecord('notifications', {uid: uid});
res.send({success: success});
});
app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
res.send({success: success});
});
// rss feed
app.get('/api/rss', async function (req, res) {
if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
res.sendStatus(403);
return;
}
// these are returned
const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
const file_type_filter = req.query.file_type_filter;
const favorite_filter = req.query.favorite_filter === 'true';
const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
const feed = new Feed({
title: 'Downloads',
description: 'YoutubeDL-Material downloads',
id: utils.getBaseURL(),
link: utils.getBaseURL(),
image: 'https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
favicon: 'https://raw.githubusercontent.com/Tzahi12345/YoutubeDL-Material/master/src/favicon.ico',
generator: 'YoutubeDL-Material'
});
files.forEach(file => {
feed.addItem({
title: file.title,
link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
description: file.description,
author: [
{
name: file.uploader,
link: file.url
}
],
contributor: [],
date: file.timestamp,
// https://stackoverflow.com/a/45415677/8088021
image: file.thumbnailURL.replace('&', '&amp;')
});
});
res.send(feed.rss2());
});
// web server
app.use(function(req, res, next) {
//if the request is not html then move along
var accept = req.accepts('html', 'json', 'xml');

91
backend/archive.js Normal file
View File

@@ -0,0 +1,91 @@
const path = require('path');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const db_api = require('./db');
exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
const filter = {user_uid: user_uid, sub_id: sub_id};
if (type) filter['type'] = type;
const archive_items = await db_api.getRecords('archives', filter);
const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
return archive_item_lines.join('\n');
}
exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => {
const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id);
const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
return success;
}
exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return success;
}
exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return !!archive_item;
}
exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = null) => {
let archive_import_count = 0;
const lines = archive_text.split('\n');
for (let line of lines) {
const archive_line_parts = line.trim().split(' ');
// should just be the extractor and the video ID
if (archive_line_parts.length !== 2) {
continue;
}
const extractor = archive_line_parts[0];
const id = archive_line_parts[1];
if (!extractor || !id) continue;
// we can't do a bulk write because we need to avoid duplicate archive items existing in db
const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id);
await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type, sub_id: sub_id, user_uid: user_uid});
archive_import_count++;
}
return archive_import_count;
}
exports.importArchives = async () => {
const imported_archives = [];
const dirs_to_check = await db_api.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db
for (let i = 0; i < dirs_to_check.length; i++) {
const dir_to_check = dirs_to_check[i];
if (!dir_to_check['archive_path']) continue;
const files_to_import = [
path.join(dir_to_check['archive_path'], `archive_${dir_to_check['type']}.txt`),
path.join(dir_to_check['archive_path'], `blacklist_${dir_to_check['type']}.txt`)
]
for (const file_to_import of files_to_import) {
const file_exists = await fs.pathExists(file_to_import);
if (!file_exists) continue;
const archive_text = await fs.readFile(file_to_import, 'utf8');
await exports.importArchiveFile(archive_text, dir_to_check.type, dir_to_check.user_uid, dir_to_check.sub_id);
imported_archives.push(file_to_import);
}
}
return imported_archives;
}
const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => {
return {
extractor: extractor,
id: id,
type: type,
title: title,
user_uid: user_uid ? user_uid : null,
sub_id: sub_id ? sub_id : null,
timestamp: Date.now() / 1000,
uid: uuid()
}
}

7
backend/authentication/auth.js
View File

@@ -33,7 +33,14 @@ exports.initialize = function () {
saltRounds = 10;
// Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
if (!(+JWT_EXPIRATION)) {
logger.warn(`JWT expiration value improperly set to ${JWT_EXPIRATION}, auto setting to 1 day.`);
JWT_EXPIRATION = 86400;
} else {
JWT_EXPIRATION = +JWT_EXPIRATION;
}
SERVER_SECRET = null;
if (db_api.users_db.get('jwt_secret').value()) {

23
backend/config.js
View File

@@ -185,7 +185,6 @@ const DEFAULT_CONFIG = {
"default_file_output": "",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true,
"max_concurrent_downloads": 5,
@@ -196,21 +195,31 @@ const DEFAULT_CONFIG = {
"file_manager_enabled": true,
"allow_quality_select": true,
"download_only_mode": false,
"allow_autoplay": true,
"force_autoplay": false,
"enable_downloads_manager": true,
"allow_playlist_categorization": true
"allow_playlist_categorization": true,
"enable_notifications": true,
"enable_all_notifications": true,
"allowed_notification_types": [],
"enable_rss_feed": false,
},
"API": {
"use_API_key": false,
"API_key": "",
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
"generate_NFO_files": false,
"use_ntfy_API": false,
"ntfy_topic_URL": "",
"use_gotify_API": false,
"gotify_server_URL": "",
"gotify_app_token": "",
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"webhook_URL": ""
},
"Themes": {
"default_theme": "default",

76
backend/consts.js
View File

@@ -30,10 +30,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_custom_args',
'path': 'YoutubeDLMaterial.Downloader.custom_args'
},
'ytdl_safe_download_override': {
'key': 'ytdl_safe_download_override',
'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
},
'ytdl_include_thumbnail': {
'key': 'ytdl_include_thumbnail',
'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
@@ -68,9 +64,9 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_download_only_mode',
'path': 'YoutubeDLMaterial.Extra.download_only_mode'
},
'ytdl_allow_autoplay': {
'key': 'ytdl_allow_autoplay',
'path': 'YoutubeDLMaterial.Extra.allow_autoplay'
'ytdl_force_autoplay': {
'key': 'ytdl_force_autoplay',
'path': 'YoutubeDLMaterial.Extra.force_autoplay'
},
'ytdl_enable_downloads_manager': {
'key': 'ytdl_enable_downloads_manager',
@@ -80,6 +76,22 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_allow_playlist_categorization',
'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
},
'ytdl_enable_notifications': {
'key': 'ytdl_enable_notifications',
'path': 'YoutubeDLMaterial.Extra.enable_notifications'
},
'ytdl_enable_all_notifications': {
'key': 'ytdl_enable_all_notifications',
'path': 'YoutubeDLMaterial.Extra.enable_all_notifications'
},
'ytdl_allowed_notification_types': {
'key': 'ytdl_allowed_notification_types',
'path': 'YoutubeDLMaterial.Extra.allowed_notification_types'
},
'ytdl_enable_rss_feed': {
'key': 'ytdl_enable_rss_feed',
'path': 'YoutubeDLMaterial.Extra.enable_rss_feed'
},
// API
'ytdl_use_api_key': {
@@ -98,18 +110,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_youtube_api_key',
'path': 'YoutubeDLMaterial.API.youtube_API_key'
},
'ytdl_use_twitch_api': {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_client_id': {
'key': 'ytdl_twitch_client_id',
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
},
'ytdl_twitch_client_secret': {
'key': 'ytdl_twitch_client_secret',
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
@@ -122,6 +122,42 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_generate_nfo_files',
'path': 'YoutubeDLMaterial.API.generate_NFO_files'
},
'ytdl_use_ntfy_API': {
'key': 'ytdl_use_ntfy_API',
'path': 'YoutubeDLMaterial.API.use_ntfy_API'
},
'ytdl_ntfy_topic_url': {
'key': 'ytdl_ntfy_topic_url',
'path': 'YoutubeDLMaterial.API.ntfy_topic_URL'
},
'ytdl_use_gotify_API': {
'key': 'ytdl_use_gotify_API',
'path': 'YoutubeDLMaterial.API.use_gotify_API'
},
'ytdl_gotify_server_url': {
'key': 'ytdl_gotify_server_url',
'path': 'YoutubeDLMaterial.API.gotify_server_URL'
},
'ytdl_gotify_app_token': {
'key': 'ytdl_gotify_app_token',
'path': 'YoutubeDLMaterial.API.gotify_app_token'
},
'ytdl_use_telegram_API': {
'key': 'ytdl_use_telegram_API',
'path': 'YoutubeDLMaterial.API.use_telegram_API'
},
'ytdl_telegram_bot_token': {
'key': 'ytdl_telegram_bot_token',
'path': 'YoutubeDLMaterial.API.telegram_bot_token'
},
'ytdl_telegram_chat_id': {
'key': 'ytdl_telegram_chat_id',
'path': 'YoutubeDLMaterial.API.telegram_chat_id'
},
'ytdl_webhook_url': {
'key': 'ytdl_webhook_url',
'path': 'YoutubeDLMaterial.API.webhook_URL'
},
// Themes
@@ -306,4 +342,4 @@ const YTDL_ARGS_WITH_VALUES = [
// we're using a Set here for performance
exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
exports.CURRENT_VERSION = 'v4.3';
exports.CURRENT_VERSION = 'v4.3.1';

116
backend/db.js
View File

@@ -2,6 +2,7 @@ var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config');
var utils = require('./utils')
@@ -58,6 +59,13 @@ const tables = {
name: 'tasks',
primary_key: 'key'
},
notifications: {
name: 'notifications',
primary_key: 'uid'
},
archives: {
name: 'archives'
},
test: {
name: 'test'
}
@@ -148,6 +156,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
await database.collection(table).createIndex(text_search);
}
});
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
return true;
} catch(err) {
logger.error(err);
@@ -252,13 +261,16 @@ exports.getFileDirectoriesAndDBs = async () => {
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'audio'),
user_uid: user.uid,
type: 'audio'
type: 'audio',
archive_path: utils.getArchiveFolder('audio', user.uid)
});
// add user's video dir to check list
dirs_to_check.push({
basePath: path.join(usersFileFolder, user.uid, 'video'),
type: 'video'
user_uid: user.uid,
type: 'video',
archive_path: utils.getArchiveFolder('video', user.uid)
});
}
} else {
@@ -268,13 +280,15 @@ exports.getFileDirectoriesAndDBs = async () => {
// add audio dir to check list
dirs_to_check.push({
basePath: audioFolderPath,
type: 'audio'
type: 'audio',
archive_path: utils.getArchiveFolder('audio')
});
// add video dir to check list
dirs_to_check.push({
basePath: videoFolderPath,
type: 'video'
type: 'video',
archive_path: utils.getArchiveFolder('video')
});
}
@@ -295,7 +309,8 @@ exports.getFileDirectoriesAndDBs = async () => {
: path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name),
user_uid: subscription_to_check.user_uid,
type: subscription_to_check.type,
sub_id: subscription_to_check['id']
sub_id: subscription_to_check['id'],
archive_path: utils.getArchiveFolder(subscription_to_check.type, subscription_to_check.user_uid, subscription_to_check)
});
}
@@ -350,7 +365,7 @@ exports.addMetadataPropertyToDB = async (property_key) => {
}
}
return await exports.bulkUpdateRecords('files', 'uid', update_obj);
return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj);
} catch(err) {
logger.error(err);
return false;
@@ -447,8 +462,8 @@ exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null)
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid, uuid);
exports.deleteFile = async (uid, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
@@ -494,16 +509,22 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = utils.getArchiveFolder(type, uuid);
// get id/extractor from JSON
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null;
if (jsonobj) id = jsonobj.id;
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
}
if (jsonExists) await fs.unlink(jsonPath);
@@ -534,8 +555,32 @@ exports.getVideo = async (file_uid) => {
return await exports.getRecord('files', {uid: file_uid});
}
exports.getFiles = async (uuid = null) => {
return await exports.getRecords('files', {user_uid: uuid});
exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => {
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
if (favorite_filter) {
filter_obj['favorite'] = true;
}
if (sub_id) {
filter_obj['sub_id'] = sub_id;
}
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search)));
const file_count = await exports.getRecords('files', filter_obj, true);
return {files, file_count};
}
exports.setVideoProperty = async (file_uid, assignment_obj) => {
@@ -550,7 +595,7 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override
if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
if (replaceFilter) local_db.get(table).remove((doc) => _.isMatch(doc, replaceFilter)).write();
local_db.get(table).push(doc).write();
return true;
}
@@ -653,9 +698,15 @@ exports.getRecords = async (table, filter_obj = null, return_count = false, sort
// Update
exports.updateRecord = async (table, filter_obj, update_obj) => {
exports.updateRecord = async (table, filter_obj, update_obj, nested_mode = false) => {
// local db override
if (using_local_db) {
if (nested_mode) {
// if object is nested we need to handle it differently
update_obj = utils.convertFlatObjectToNestedObject(update_obj);
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').merge(update_obj).write();
return true;
}
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').assign(update_obj).write();
return true;
}
@@ -677,7 +728,19 @@ exports.updateRecords = async (table, filter_obj, update_obj) => {
return !!(output['result']['ok']);
}
exports.bulkUpdateRecords = async (table, key_label, update_obj) => {
exports.removePropertyFromRecord = async (table, filter_obj, remove_obj) => {
// local db override
if (using_local_db) {
const props_to_remove = Object.keys(remove_obj);
exports.applyFilterLocalDB(local_db.get(table), filter_obj, 'find').unset(props_to_remove).write();
return true;
}
const output = await database.collection(table).updateOne(filter_obj, {$unset: remove_obj});
return !!(output['result']['ok']);
}
exports.bulkUpdateRecordsByKey = async (table, key_label, update_obj) => {
// local db override
if (using_local_db) {
local_db.get(table).each((record) => {
@@ -1091,6 +1154,14 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
} else if ('$ne' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] !== filter_prop_value['$ne'];
} else if ('$lt' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] < filter_prop_value['$lt'];
} else if ('$gt' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] > filter_prop_value['$gt'];
} else if ('$lte' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] <= filter_prop_value['$lt'];
} else if ('$gte' in filter_prop_value) {
filtered &= filter_prop in record && record[filter_prop] >= filter_prop_value['$gt'];
}
} else {
// handle case of nested property check
@@ -1105,3 +1176,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
});
return return_val;
}
// should only be used for tests
exports.setLocalDBMode = (mode) => {
using_local_db = mode;
}

125
backend/downloader.js
View File

@@ -1,7 +1,6 @@
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const path = require('path');
const mergeFiles = require('merge-files');
const NodeID3 = require('node-id3')
const Mutex = require('async-mutex').Mutex;
@@ -14,6 +13,8 @@ const { create } = require('xmlbuilder2');
const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const notifications_api = require('./notifications');
const archive_api = require('./archive');
const mutex = new Mutex();
let should_check_downloads = true;
@@ -26,6 +27,25 @@ if (db_api.database_initialized) {
});
}
/*
This file handles all the downloading functionality.
To download a file, we go through 4 steps. Here they are with their respective index & function:
0: Create the download
- createDownload()
1: Get info for the download (we need this step for categories and archive functionality)
- collectInfo()
2: Download the file
- downloadQueuedFile()
3: Complete
- N/A
We use checkDownloads() to move downloads through the steps and call their respective functions.
*/
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
return await mutex.runExclusive(async () => {
const download = {
@@ -84,10 +104,10 @@ exports.resumeDownload = async (download_uid) => {
exports.restartDownload = async (download_uid) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await exports.clearDownload(download_uid);
const success = !!(await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']));
const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
should_check_downloads = true;
return success;
return new_download;
}
exports.cancelDownload = async (download_uid) => {
@@ -106,9 +126,10 @@ exports.clearDownload = async (download_uid) => {
return await db_api.removeRecord('download_queue', {uid: download_uid});
}
async function handleDownloadError(download_uid, error_message) {
if (!download_uid) return;
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error_message, finished: true, running: false});
async function handleDownloadError(download, error_message, error_type = null) {
if (!download || !download['uid']) return;
notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_type);
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
}
async function setupDownloads() {
@@ -154,6 +175,13 @@ async function checkDownloads() {
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
if (waiting_download['finished_step'] && !waiting_download['finished']) {
if (waiting_download['sub_id']) {
const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
if (sub_missing) {
handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
continue;
}
}
// move to next step
running_downloads_count++;
if (waiting_download['step_index'] === 0) {
@@ -193,6 +221,20 @@ async function collectInfo(download_uid) {
return;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
if (exists_in_archive) {
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
logger.warn(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error, 'exists_in_archive');
return;
}
}
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
@@ -272,14 +314,14 @@ async function downloadQueuedFile(download_uid) {
clearInterval(download_checker);
if (err) {
logger.error(err.stderr);
await handleDownloadError(download_uid, err.stderr);
await handleDownloadError(download, err.stderr);
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download_uid, error_message);
await handleDownloadError(download, error_message, 'no_output');
logger.warn(error_message);
resolve(false);
return;
@@ -288,7 +330,10 @@ async function downloadQueuedFile(download_uid) {
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
// we have to do this because sometimes there will be leading characters before the actual json
const start_idx = output[i].indexOf('{"');
const clean_output = output[i].slice(start_idx, output[i].length);
output_json = JSON.parse(clean_output);
} catch(e) {
output_json = null;
}
@@ -305,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
@@ -341,15 +386,12 @@ async function downloadQueuedFile(download_uid) {
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj);
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj);
}
let container = null;
@@ -363,7 +405,7 @@ async function downloadQueuedFile(download_uid) {
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download_uid, error_message);
await handleDownloadError(download, error_message, 'no_metadata');
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
@@ -379,6 +421,10 @@ async function downloadQueuedFile(download_uid) {
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
}
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
@@ -426,7 +472,8 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
} else if (heightParam && heightParam !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height${maxHeight ? '<' : ''}=${heightParam}]`];
const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
@@ -463,28 +510,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
@@ -560,7 +585,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
await handleDownloadError(download_uid, error);
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error, 'parse_failed');
}
resolve(null);
}
@@ -569,7 +595,8 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
if (err.stderr) error_message += `\n\n${err.stderr}`;
logger.error(error_message);
if (download_uid) {
await handleDownloadError(download_uid, error_message);
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error_message);
}
resolve(null);
}
@@ -635,13 +662,3 @@ exports.generateNFOFile = (info, output_path) => {
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join('appdata', 'archives');
}
}

2
backend/entrypoint.sh
View File

@@ -10,7 +10,7 @@ fi
# chown current working directory to current user
if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos."
exec gosu "$UID:$GID" "$0" "$@"
fi

157
backend/notifications.js Normal file
View File

@@ -0,0 +1,157 @@
const db_api = require('./db');
const config_api = require('./config');
const logger = require('./logger');
const utils = require('./utils');
const { uuid } = require('uuidv4');
const fetch = require('node-fetch');
const { gotify } = require("gotify");
const TelegramBot = require('node-telegram-bot-api');
const NOTIFICATION_TYPE_TO_TITLE = {
task_finished: 'Task finished',
download_complete: 'Download complete',
download_error: 'Download error'
}
const NOTIFICATION_TYPE_TO_BODY = {
task_finished: (notification) => notification['data']['task_title'],
download_complete: (notification) => {return `${notification['data']['file_title']}\nOriginal URL: ${notification['data']['original_url']}`},
download_error: (notification) => {return `Error: ${notification['data']['download_error_type']}\nURL: ${notification['data']['download_url']}`}
}
const NOTIFICATION_TYPE_TO_URL = {
task_finished: () => {return `${utils.getBaseURL()}/#/tasks`},
download_complete: (notification) => {return `${utils.getBaseURL()}/#/player;uid=${notification['data']['file_uid']}`},
download_error: () => {return `${utils.getBaseURL()}/#/downloads`},
}
const NOTIFICATION_TYPE_TO_THUMBNAIL = {
task_finished: () => null,
download_complete: (notification) => notification['data']['file_thumbnail'],
download_error: () => null
}
exports.sendNotification = async (notification) => {
// info necessary if we are using 3rd party APIs
const type = notification['type'];
const data = {
title: NOTIFICATION_TYPE_TO_TITLE[type],
body: NOTIFICATION_TYPE_TO_BODY[type](notification),
type: type,
url: NOTIFICATION_TYPE_TO_URL[type](notification),
thumbnail: NOTIFICATION_TYPE_TO_THUMBNAIL[type](notification)
}
if (config_api.getConfigItem('ytdl_use_ntfy_API') && config_api.getConfigItem('ytdl_ntfy_topic_url')) {
sendNtfyNotification(data);
}
if (config_api.getConfigItem('ytdl_use_gotify_API') && config_api.getConfigItem('ytdl_gotify_server_url') && config_api.getConfigItem('ytdl_gotify_app_token')) {
sendGotifyNotification(data);
}
if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
sendTelegramNotification(data);
}
if (config_api.getConfigItem('ytdl_webhook_url')) {
sendGenericNotification(data);
}
await db_api.insertRecordIntoTable('notifications', notification);
return notification;
}
exports.sendTaskNotification = async (task_obj, confirmed) => {
if (!notificationEnabled('task_finished')) return;
// workaround for tasks which are user_uid agnostic
const user_uid = config_api.getConfigItem('ytdl_multi_user_mode') ? 'admin' : null;
await db_api.removeAllRecords('notifications', {"data.task_key": task_obj.key});
const data = {task_key: task_obj.key, task_title: task_obj.title, confirmed: confirmed};
const notification = exports.createNotification('task_finished', ['view_tasks'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.sendDownloadNotification = async (file, user_uid) => {
if (!notificationEnabled('download_complete')) return;
const data = {file_uid: file.uid, file_title: file.title, file_thumbnail: file.thumbnailURL, original_url: file.url};
const notification = exports.createNotification('download_complete', ['play'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.sendDownloadErrorNotification = async (download, user_uid, error_type = null) => {
if (!notificationEnabled('download_error')) return;
const data = {download_uid: download.uid, download_url: download.url, download_error_type: error_type};
const notification = exports.createNotification('download_error', ['view_download_error', 'retry_download'], data, user_uid);
return await exports.sendNotification(notification);
}
exports.createNotification = (type, actions, data, user_uid) => {
const notification = {
type: type,
actions: actions,
data: data,
user_uid: user_uid,
uid: uuid(),
read: false,
timestamp: Date.now()/1000
}
return notification;
}
function notificationEnabled(type) {
return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
}
function sendNtfyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to ntfy');
fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
method: 'POST',
body: body,
headers: {
'Title': title,
'Tags': type,
'Click': url,
'Attach': thumbnail
}
});
}
async function sendGotifyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to gotify');
await gotify({
server: config_api.getConfigItem('ytdl_gotify_server_url'),
app: config_api.getConfigItem('ytdl_gotify_app_token'),
title: title,
message: body,
tag: type,
priority: 5, // Keeping default from docs, may want to change this,
extras: {
"client::notification": {
click: { url: url },
bigImageUrl: thumbnail
}
}
});
}
async function sendTelegramNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to Telegram');
const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
const bot = new TelegramBot(bot_token);
if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
}
function sendGenericNotification(data) {
const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
logger.verbose(`Sending generic notification to ${webhook_url}`);
fetch(webhook_url, {
method: 'POST',
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data),
});
}

519
backend/package-lock.json generated
View File

@@ -64,6 +64,14 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.6.0.tgz",
"integrity": "sha512-mikldZQitV94akrc4sCcSjtJfsTKt4p+e/s0AGscVA6XArQ9kFclP+ZiYUMnq987rc6QlYxXv/EivqlfSLxpKA=="
},
"@types/responselike": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
"integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
"requires": {
"@types/node": "*"
}
},
"@ungap/promise-all-settled": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
@@ -144,7 +152,7 @@
"append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw=="
"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
},
"archiver": {
"version": "5.3.1",
@@ -198,11 +206,31 @@
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
},
"array-buffer-byte-length": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz",
"integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==",
"requires": {
"call-bind": "^1.0.2",
"is-array-buffer": "^3.0.1"
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"array.prototype.findindex": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/array.prototype.findindex/-/array.prototype.findindex-2.2.1.tgz",
"integrity": "sha512-tMj4uTmGpaGUh4XFMUh3H7KYAIqlygrlXchOEVTiICbTwRwMhDqtzsOwvtI+WAf1GdjJBeIP3Bu92Qg0SnXdtA==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.4",
"es-abstract": "^1.20.4",
"es-shim-unscopables": "^1.0.0"
}
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -239,6 +267,11 @@
"resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
"integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg=="
},
"available-typed-arrays": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
"integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@@ -420,6 +453,29 @@
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
},
"cacheable-lookup": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz",
"integrity": "sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg==",
"requires": {
"@types/keyv": "^3.1.1",
"keyv": "^4.0.0"
}
},
"cacheable-request": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz",
"integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==",
"requires": {
"clone-response": "^1.0.2",
"get-stream": "^5.1.0",
"http-cache-semantics": "^4.0.0",
"keyv": "^4.0.0",
"lowercase-keys": "^2.0.0",
"normalize-url": "^6.0.1",
"responselike": "^2.0.0"
}
},
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
@@ -467,6 +523,21 @@
}
}
},
"clone-response": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
"integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==",
"requires": {
"mimic-response": "^1.0.0"
},
"dependencies": {
"mimic-response": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="
}
}
},
"color": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
@@ -745,6 +816,11 @@
}
}
},
"duplexer3": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz",
"integrity": "sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA=="
},
"ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1015,6 +1091,14 @@
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
"integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA=="
},
"for-each": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
"integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
"requires": {
"is-callable": "^1.1.3"
}
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -1135,6 +1219,64 @@
"is-glob": "^4.0.1"
}
},
"globalthis": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz",
"integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==",
"requires": {
"define-properties": "^1.1.3"
}
},
"gopd": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
"requires": {
"get-intrinsic": "^1.1.3"
},
"dependencies": {
"get-intrinsic": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz",
"integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==",
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.3"
}
}
}
},
"got": {
"version": "10.7.0",
"resolved": "https://registry.npmjs.org/got/-/got-10.7.0.tgz",
"integrity": "sha512-aWTDeNw9g+XqEZNcTjMMZSy7B7yE9toWOFYip7ofFTLleJhvZwUxxTxkTpKvF+p1SAA4VHmuEy7PiHTHyq8tJg==",
"requires": {
"@sindresorhus/is": "^2.0.0",
"@szmarczak/http-timer": "^4.0.0",
"@types/cacheable-request": "^6.0.1",
"cacheable-lookup": "^2.0.0",
"cacheable-request": "^7.0.1",
"decompress-response": "^5.0.0",
"duplexer3": "^0.1.4",
"get-stream": "^5.0.0",
"lowercase-keys": "^2.0.0",
"mimic-response": "^2.1.0",
"p-cancelable": "^2.0.0",
"p-event": "^4.0.0",
"responselike": "^2.0.0",
"to-readable-stream": "^2.0.0",
"type-fest": "^0.10.0"
}
},
"gotify": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/gotify/-/gotify-1.1.0.tgz",
"integrity": "sha512-f3PUh08i+1JCJuzv9CRYuIx1QOb9DoWXRvWaPttgiZEG7XtetDQVv6S9cr6suPNKkZI4ry8dqdU2qdb9AiPinw==",
"requires": {
"got": "10.7.0"
}
},
"graceful-fs": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
@@ -1175,11 +1317,24 @@
"get-intrinsic": "^1.1.1"
}
},
"has-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
"integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg=="
},
"has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
},
"has-tostringtag": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
"requires": {
"has-symbols": "^1.0.2"
}
},
"hashish": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz",
@@ -1201,6 +1356,11 @@
"zero-fill": "^2.2.3"
}
},
"http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
"http-errors": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
@@ -1260,11 +1420,41 @@
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
},
"is-array-buffer": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz",
"integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==",
"requires": {
"call-bind": "^1.0.2",
"get-intrinsic": "^1.2.0",
"is-typed-array": "^1.1.10"
},
"dependencies": {
"get-intrinsic": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz",
"integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==",
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.3"
}
}
}
},
"is-arrayish": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
"integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
},
"is-bigint": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
"integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==",
"requires": {
"has-bigints": "^1.0.1"
}
},
"is-binary-path": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
@@ -1273,6 +1463,15 @@
"binary-extensions": "^2.0.0"
}
},
"is-boolean-object": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz",
"integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==",
"requires": {
"call-bind": "^1.0.2",
"has-tostringtag": "^1.0.0"
}
},
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
@@ -1300,11 +1499,24 @@
"define-properties": "^1.1.3"
}
},
"is-negative-zero": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz",
"integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA=="
},
"is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
},
"is-number-object": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz",
"integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==",
"requires": {
"has-tostringtag": "^1.0.0"
}
},
"is-plain-obj": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz",
@@ -1315,11 +1527,56 @@
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
"integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
},
"is-regex": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
"integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==",
"requires": {
"call-bind": "^1.0.2",
"has-tostringtag": "^1.0.0"
}
},
"is-shared-array-buffer": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz",
"integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==",
"requires": {
"call-bind": "^1.0.2"
}
},
"is-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
"integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
},
"is-string": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
"integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
"requires": {
"has-tostringtag": "^1.0.0"
}
},
"is-symbol": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz",
"integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==",
"requires": {
"has-symbols": "^1.0.2"
}
},
"is-typed-array": {
"version": "1.1.10",
"resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz",
"integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==",
"requires": {
"available-typed-arrays": "^1.0.5",
"call-bind": "^1.0.2",
"for-each": "^0.3.3",
"gopd": "^1.0.1",
"has-tostringtag": "^1.0.0"
}
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@@ -1330,6 +1587,14 @@
"resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
"integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw=="
},
"is-weakref": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz",
"integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==",
"requires": {
"call-bind": "^1.0.2"
}
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
@@ -1358,6 +1623,11 @@
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -1369,12 +1639,9 @@
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"json5": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.1.2.tgz",
"integrity": "sha512-MoUOQ4WdiN3yxhm7NEVJSJrieAo5hNSLQ5sj05OTRHPL9HOBy8u4Bu88jsC1jvqAdN+E1bJmsUcZH+1HQxliqQ==",
"requires": {
"minimist": "^1.2.5"
}
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.2.tgz",
"integrity": "sha512-46Tk9JiOL2z7ytNQWFLpj99RZkVgeHf87yGQKsIkaPz1qSH9UczKH1rO7K3wgRselo0tYMUNfecYpm/p1vC7tQ=="
},
"jsonfile": {
"version": "6.0.1",
@@ -1446,6 +1713,14 @@
"safe-buffer": "^5.0.1"
}
},
"keyv": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.2.tgz",
"integrity": "sha512-5MHbFaKn8cNSmVW7BYnijeAVlE4cYA/SVkifVgrh7yotnfhKmjuXpDKjrABLnT0SfHWV21P8ow07OGfRrNDg8g==",
"requires": {
"json-buffer": "3.0.1"
}
},
"kuler": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz",
@@ -1651,6 +1926,11 @@
"steno": "^0.4.1"
}
},
"lowercase-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
"integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -1690,14 +1970,6 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
},
"merge-files": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/merge-files/-/merge-files-0.1.2.tgz",
"integrity": "sha512-WTvtH6ZwVy1/scvp1M+Re6PVni87QTjpSLAwxh0L+PlYIxc4VGFFpLjvP7jdJ43gaJ5n+RUIriJ6wKqmqvVVmg==",
"requires": {
"multistream": "^2.1.0"
}
},
"merge-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -1731,10 +2003,15 @@
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
},
"mimic-response": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"requires": {
"brace-expansion": "^1.1.7"
}
@@ -1883,9 +2160,9 @@
}
},
"moment": {
"version": "2.29.2",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz",
"integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg=="
"version": "2.29.4",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
},
"mongodb": {
"version": "3.7.3",
@@ -2038,11 +2315,27 @@
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-inspect": {
"version": "1.12.3",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
"integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g=="
},
"object-keys": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
},
"object.assign": {
"version": "4.1.4",
"resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz",
"integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.4",
"has-symbols": "^1.0.3",
"object-keys": "^1.1.1"
}
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -2132,35 +2425,37 @@
}
},
"passport-jwt": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/passport-jwt/-/passport-jwt-4.0.0.tgz",
"integrity": "sha512-BwC0n2GP/1hMVjR4QpnvqA61TxenUMlmfNjYNgK0ZAs0HK4SOQkHcSv4L328blNTLtHq7DbmvyNJiH+bn6C5Mg==",
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/passport-jwt/-/passport-jwt-4.0.1.tgz",
"integrity": "sha512-UCKMDYhNuGOBE9/9Ycuoyh7vP6jpeTp/+sfMJl7nLff/t6dps+iaeE0hhNkKN8/HZHcJ7lCdOyDxHdDoxoSvdQ==",
"requires": {
"jsonwebtoken": "^8.2.0",
"jsonwebtoken": "^9.0.0",
"passport-strategy": "^1.0.0"
},
"dependencies": {
"jsonwebtoken": {
"version": "8.5.1",
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
"integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
"integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
"requires": {
"jws": "^3.2.2",
"lodash.includes": "^4.3.0",
"lodash.isboolean": "^3.0.3",
"lodash.isinteger": "^4.0.4",
"lodash.isnumber": "^3.0.3",
"lodash.isplainobject": "^4.0.6",
"lodash.isstring": "^4.0.1",
"lodash.once": "^4.0.0",
"lodash": "^4.17.21",
"ms": "^2.1.1",
"semver": "^5.6.0"
"semver": "^7.3.8"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"requires": {
"lru-cache": "^6.0.0"
}
}
}
},
@@ -2383,6 +2678,32 @@
}
}
},
"request-promise": {
"version": "4.2.6",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.6.tgz",
"integrity": "sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==",
"requires": {
"bluebird": "^3.5.0",
"request-promise-core": "1.1.4",
"stealthy-require": "^1.1.1",
"tough-cookie": "^2.3.3"
},
"dependencies": {
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
}
}
},
"request-promise-core": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz",
"integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==",
"requires": {
"lodash": "^4.17.19"
}
},
"require-at": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/require-at/-/require-at-1.0.6.tgz",
@@ -2421,6 +2742,28 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"safe-regex-test": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz",
"integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==",
"requires": {
"call-bind": "^1.0.2",
"get-intrinsic": "^1.1.3",
"is-regex": "^1.1.4"
},
"dependencies": {
"get-intrinsic": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz",
"integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==",
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.3"
}
}
}
},
"safe-stable-stringify": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz",
@@ -2522,6 +2865,16 @@
"nanoid": "^2.1.0"
}
},
"side-channel": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
"integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"requires": {
"call-bind": "^1.0.0",
"get-intrinsic": "^1.0.2",
"object-inspect": "^1.9.0"
}
},
"signal-exit": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
@@ -2580,6 +2933,11 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
},
"stealthy-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
"integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g=="
},
"steno": {
"version": "0.4.4",
"resolved": "https://registry.npmjs.org/steno/-/steno-0.4.4.tgz",
@@ -2636,6 +2994,36 @@
}
}
},
"string.prototype.trim": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz",
"integrity": "sha512-p6TmeT1T3411M8Cgg9wBTMRtY2q9+PNy9EV1i2lIXUN/btt763oIfxwN3RR8VU6wHX8j/1CFy0L+YuThm6bgOg==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.4",
"es-abstract": "^1.20.4"
}
},
"string.prototype.trimend": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz",
"integrity": "sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.4",
"es-abstract": "^1.20.4"
}
},
"string.prototype.trimstart": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz",
"integrity": "sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.4",
"es-abstract": "^1.20.4"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
@@ -2690,6 +3078,11 @@
"thenify": ">= 3.1.0 < 4"
}
},
"to-readable-stream": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz",
"integrity": "sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w=="
},
"to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -2754,10 +3147,31 @@
"mime-types": "~2.1.24"
}
},
"typed-array-length": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz",
"integrity": "sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==",
"requires": {
"call-bind": "^1.0.2",
"for-each": "^0.3.3",
"is-typed-array": "^1.1.9"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"unbox-primitive": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz",
"integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==",
"requires": {
"call-bind": "^1.0.2",
"has-bigints": "^1.0.2",
"has-symbols": "^1.0.3",
"which-boxed-primitive": "^1.0.2"
}
},
"uid-safe": {
"version": "2.1.5",
@@ -2886,6 +3300,31 @@
"isexe": "^2.0.0"
}
},
"which-boxed-primitive": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz",
"integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==",
"requires": {
"is-bigint": "^1.0.1",
"is-boolean-object": "^1.1.0",
"is-number-object": "^1.0.4",
"is-string": "^1.0.5",
"is-symbol": "^1.0.3"
}
},
"which-typed-array": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz",
"integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==",
"requires": {
"available-typed-arrays": "^1.0.5",
"call-bind": "^1.0.2",
"for-each": "^0.3.3",
"gopd": "^1.0.1",
"has-tostringtag": "^1.0.0",
"is-typed-array": "^1.1.10"
}
},
"winston": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/winston/-/winston-3.7.2.tgz",

9
backend/package.json
View File

@@ -28,22 +28,25 @@
"config": "^3.2.3",
"express": "^4.17.3",
"express-session": "^1.17.3",
"feed": "^4.2.2",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"gotify": "^1.1.0",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^9.2.2",
"moment": "^2.29.2",
"moment": "^2.29.4",
"mongodb": "^3.6.9",
"multer": "1.4.5-lts.1",
"node-fetch": "^2.6.7",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"node-telegram-bot-api": "^0.61.0",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",
"passport-jwt": "^4.0.1",
"passport-ldapauth": "^3.0.1",
"passport-local": "^1.0.0",
"progress": "^2.0.3",

61
backend/subscriptions.js
View File

@@ -3,6 +3,7 @@ const path = require('path');
const youtubedl = require('youtube-dl');
const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
@@ -138,28 +139,25 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && (await fs.pathExists(appendedBasePath))) {
if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists
// TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
if (await fs.pathExists(archive_file_path)) {
await fs.unlink(archive_file_path);
}
await fs.rmdir(sub.archive);
}
await fs.remove(appendedBasePath);
}
await db_api.removeAllRecords('archives', {sub_id: sub.id});
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
if (typeof sub === 'string') {
// TODO: fix bad workaround where sub is a sub_id
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
}
// TODO: combine this with deletefile
let basePath = null;
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
let retrievedExtractor = null;
await db_api.removeRecord('files', {uid: file_uid});
@@ -178,7 +176,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
]);
if (jsonExists) {
retrievedID = fs.readJSONSync(jsonPath)['id'];
const info_json = fs.readJSONSync(jsonPath);
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
await fs.unlink(jsonPath);
}
@@ -196,11 +196,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
return false;
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
if (useArchive && retrievedID) {
const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
}
return true;
}
@@ -331,8 +329,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = getAppendedBasePath(sub, basePath);
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -368,21 +364,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...customArgsArray);
}
let archive_dir = null;
let archive_path = null;
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
if (sub.type && sub.type === 'audio') {
archive_path = path.join(archive_dir, 'merged_audio.txt');
} else {
archive_path = path.join(archive_dir, 'merged_video.txt');
}
}
downloadConfig.push('--download-archive', archive_path);
}
if (sub.timerange && !redownload) {
downloadConfig.push('--dateafter', sub.timerange);
}
@@ -425,7 +406,14 @@ async function getFilesToDownload(sub, output_jsons) {
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
}
files_to_download.push(output_json);
}
}
@@ -444,7 +432,12 @@ async function getAllSubscriptions() {
}
async function getSubscription(subID) {
return await db_api.getRecord('subscriptions', {id: subID});
// stringify and parse because we may override the 'downloading' property
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
// now with the download_queue, we may need to override 'downloading'
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true);
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
return sub;
}
async function getSubscriptionByName(subName, user_uid = null) {

79
backend/tasks.js
View File

@@ -1,5 +1,7 @@
const db_api = require('./db');
const notifications_api = require('./notifications');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const fs = require('fs-extra');
const logger = require('./logger');
@@ -33,6 +35,28 @@ const TASKS = {
confirm: youtubedl_api.updateYoutubeDL,
title: 'Update youtube-dl',
job: null
},
delete_old_files: {
run: checkForAutoDeleteFiles,
confirm: autoDeleteFiles,
title: 'Delete old files',
job: null
},
import_legacy_archives: {
run: archive_api.importArchives,
title: 'Import legacy archives',
job: null
}
}
const defaultOptions = {
all: {
auto_confirm: false
},
delete_old_files: {
blacklist_files: false,
blacklist_subscription_files: false,
threshold_days: ''
}
}
@@ -45,7 +69,7 @@ function scheduleJob(task_key, schedule) {
const dayOfWeek = schedule['data']['dayOfWeek'] != null ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] != null ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] != null ? schedule['data']['minute'] : null;
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute, undefined, schedule['data']['tz'] ? schedule['data']['tz'] : undefined);
} else {
logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
return null;
@@ -57,7 +81,7 @@ function scheduleJob(task_key, schedule) {
logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
return;
}
// remove schedule if it's a one-time task
if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
// we're just "running" the task, any confirmation should be user-initiated
@@ -77,9 +101,10 @@ exports.setupTasks = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
const mergedDefaultOptions = Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {});
const task_in_db = await db_api.getRecord('tasks', {key: task_key});
if (!task_in_db) {
// insert task metadata into table if missing
// insert task metadata into table if missing, eventually move title to UI
await db_api.insertRecordIntoTable('tasks', {
key: task_key,
title: TASKS[task_key]['title'],
@@ -90,9 +115,19 @@ exports.setupTasks = async () => {
data: null,
error: null,
schedule: null,
options: {}
options: Object.assign({}, defaultOptions['all'], defaultOptions[task_key] || {})
});
} else {
// verify all options exist in task
for (const key of Object.keys(mergedDefaultOptions)) {
const option_key = `options.${key}`;
// Remove any potential mangled option keys (#861)
await db_api.removePropertyFromRecord('tasks', {key: task_key}, {[option_key]: true});
if (!(task_in_db.options && task_in_db.options.hasOwnProperty(key))) {
await db_api.updateRecord('tasks', {key: task_key}, {[option_key]: mergedDefaultOptions[key]}, true);
}
}
// reset task if necessary
await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
@@ -123,15 +158,23 @@ exports.executeTask = async (task_key) => {
exports.executeRun = async (task_key) => {
logger.verbose(`Running task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {error: null})
// don't set running to true when backup up DB as it will be stick "running" if restored
if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
const data = await TASKS[task_key].run();
await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
logger.verbose(`Finished running task ${task_key}`);
const task_obj = await db_api.getRecord('tasks', {key: task_key});
await notifications_api.sendTaskNotification(task_obj, false);
if (task_obj['options'] && task_obj['options']['auto_confirm']) {
exports.executeConfirm(task_key);
}
}
exports.executeConfirm = async (task_key) => {
logger.verbose(`Confirming task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {error: null})
if (!TASKS[task_key]['confirm']) {
return null;
}
@@ -141,6 +184,7 @@ exports.executeConfirm = async (task_key) => {
await TASKS[task_key].confirm(data);
await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
logger.verbose(`Finished confirming task ${task_key}`);
await notifications_api.sendTaskNotification(task_obj, false);
}
exports.updateTaskSchedule = async (task_key, schedule) => {
@@ -193,4 +237,31 @@ async function removeDuplicates(data) {
}
}
// auto delete files
async function checkForAutoDeleteFiles() {
const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
if (!task_obj['options'] || !task_obj['options']['threshold_days']) {
const error_message = 'Failed to do delete check because no limit was set!';
logger.error(error_message);
await db_api.updateRecord('tasks', {key: 'delete_old_files'}, {error: error_message})
return null;
}
const delete_older_than_timestamp = Date.now() - task_obj['options']['threshold_days']*86400*1000;
const files = (await db_api.getRecords('files', {registered: {$lt: delete_older_than_timestamp}}))
const files_to_remove = files.map(file => {return {uid: file.uid, sub_id: file.sub_id}});
return {files_to_remove: files_to_remove};
}
async function autoDeleteFiles(data) {
const task_obj = await db_api.getRecord('tasks', {key: 'delete_old_files'});
if (data['files_to_remove']) {
logger.info(`Removing ${data['files_to_remove'].length} old files!`);
for (let i = 0; i < data['files_to_remove'].length; i++) {
const file_to_remove = data['files_to_remove'][i];
await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files']));
}
}
}
exports.TASKS = TASKS;

436
backend/test/tests.js
View File

@@ -3,7 +3,6 @@ const low = require('lowdb')
const winston = require('winston');
const path = require('path');
process.chdir('./backend')
const FileSync = require('lowdb/adapters/FileSync');
@@ -38,6 +37,7 @@ var auth_api = require('../authentication/auth');
var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const archive_api = require('../archive');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
@@ -66,12 +66,12 @@ const sample_video_json = {
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
// it('Migrate', async function() {
// await db_api.connectToDB();
// await db_api.removeAllRecords();
// const success = await db_api.importJSONToDB(db.value(), users_db.value());
// assert(success);
// });
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
@@ -104,157 +104,199 @@ describe('Database', async function() {
});
});
describe('Export', function() {
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
// test both local_db and remote_db
const local_db_modes = [false, true];
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
for (const local_db_mode of local_db_modes) {
let use_local_db = local_db_mode;
describe(`Use local DB - ${use_local_db}`, async function() {
beforeEach(async function() {
if (!use_local_db) {
this.timeout(120000);
await db_api.connectToDB(0);
}
await db_api.removeAllRecords('test');
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Add and read record - Nested property', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
assert(added_record['test_add'] === 'test');
assert(!not_added_record);
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Replace filter', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
assert(count === 1);
await db_api.removeRecord('test', {test_replace_filter: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
it('Remove property from record', async function() {
await db_api.insertRecordIntoTable('test', {test_keep: 'test', test_remove: 'test'});
await db_api.removePropertyFromRecord('test', {test_keep: 'test'}, {test_remove: true});
const updated_record = await db_api.getRecord('test', {test_keep: 'test'});
assert(updated_record['test_keep']);
assert(!updated_record['test_remove']);
await db_api.removeRecord('test', {test_keep: 'test'});
});
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Remove records', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test'});
await db_api.insertRecordIntoTable('test', {test_remove: 'test', test_property: 'test2'});
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeAllRecords('test', {test_remove: 'test'});
assert(delete_succeeded);
const count = await db_api.getRecords('test', {test_remove: 'test'}, true);
assert(count === 0);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
console.log(random_record)
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
console.log(`Query time: ${(query_end - query_start)/1000}s`);
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
success = !!random_record;
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
assert(success);
});
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
}
});
describe('Local DB Filters', async function() {
@@ -289,8 +331,6 @@ describe('Multi User', async function() {
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
});
describe('Authentication', function() {
@@ -468,7 +508,7 @@ describe('Downloader', function() {
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1493770675';
const example_vod = '1710641401';
it('Download VOD', async function() {
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
@@ -593,30 +633,72 @@ describe('Tasks', function() {
});
describe('Archive', async function() {
const archive_path = path.join('test', 'archives');
fs.ensureDirSync(archive_path);
const archive_file_path = path.join(archive_path, 'archive_video.txt');
const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
beforeEach(async function() {
if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
fs.writeFileSync(blacklist_file_path, '');
});
it('Delete from archive', async function() {
await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
const new_archive = fs.readFileSync(archive_file_path);
assert(!new_archive.includes('testing2'));
await db_api.connectToDB();
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
});
it('Delete from archive - blacklist', async function() {
await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
const new_archive = fs.readFileSync(archive_file_path);
const new_blacklist = fs.readFileSync(blacklist_file_path);
assert(!new_archive.includes('testing2'));
assert(new_blacklist.includes('testing2'));
afterEach(async function() {
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
});
it('Import archive', async function() {
const archive_text = `
testextractor1 testing1
testextractor1 testing2
testextractor2 testing1
testextractor1 testing3
`;
const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
assert(count === 4)
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
console.log(archive_items);
assert(archive_items.length === 4);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
assert(success);
});
it('Get archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(archive_item1 && archive_item2);
});
it('Archive duplicates', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
assert(count === 3);
});
it('Remove from archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
assert(success);
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
assert(!!archive_item1);
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(!archive_item2);
const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
assert(!!archive_item3);
});
});
@@ -626,4 +708,24 @@ describe('Utils', async function() {
const stripped_obj = utils.stripPropertiesFromObject(test_obj, ['test1', 'test3']);
assert(!stripped_obj['test1'] && stripped_obj['test2'] && !stripped_obj['test3'])
});
it('Convert flat object to nested object', async function() {
// No modfication
const flat_obj0 = {'test1': {'test_sub': true}, 'test2': {test_sub: true}};
const nested_obj0 = utils.convertFlatObjectToNestedObject(flat_obj0);
assert(nested_obj0['test1'] && nested_obj0['test1']['test_sub']);
assert(nested_obj0['test2'] && nested_obj0['test2']['test_sub']);
// Standard setup
const flat_obj1 = {'test1.test_sub': true, 'test2.test_sub': true};
const nested_obj1 = utils.convertFlatObjectToNestedObject(flat_obj1);
assert(nested_obj1['test1'] && nested_obj1['test1']['test_sub']);
assert(nested_obj1['test2'] && nested_obj1['test2']['test_sub']);
// Nested branches
const flat_obj2 = {'test1.test_sub': true, 'test1.test2.test_sub': true};
const nested_obj2 = utils.convertFlatObjectToNestedObject(flat_obj2);
assert(nested_obj2['test1'] && nested_obj2['test1']['test_sub']);
assert(nested_obj2['test1'] && nested_obj2['test1']['test2'] && nested_obj2['test1']['test2']['test_sub']);
});
});

25
backend/twitch.js
View File

@@ -4,19 +4,28 @@ const logger = require('./logger');
const moment = require('moment');
const fs = require('fs-extra')
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
async function getCommentsForVOD(clientID, clientSecret, vodId) {
const { promisify } = require('util');
const child_process = require('child_process');
async function getCommentsForVOD(vodId) {
const exec = promisify(child_process.exec);
// Reject invalid params to prevent command injection attack
if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
if (!vodId.match(/^[0-9a-z]+$/)) {
logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
return null;
}
const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
const is_windows = process.platform === 'win32';
const cliExt = is_windows ? '.exe' : ''
const cliPath = `TwitchDownloaderCLI${cliExt}`
if (!fs.existsSync(cliPath)) {
logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
return null;
}
const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -73,9 +82,7 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
const chat = await getCommentsForVOD(vodId);
// save file if needed params are included
let file_path = null;

208
backend/utils.js
View File

@@ -4,6 +4,7 @@ const ffmpeg = require('fluent-ffmpeg');
const archiver = require('archiver');
const fetch = require('node-fetch');
const ProgressBar = require('progress');
const winston = require('winston');
const config_api = require('./config');
const logger = require('./logger');
@@ -12,7 +13,7 @@ const CONSTS = require('./consts');
const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
exports.getTrueFileName = (unfixed_path, type) => {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -27,13 +28,13 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path;
}
async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
exports.getDownloadedFilesByType = async (basePath, type, full_metadata = false) => {
// return empty array if the path doesn't exist
if (!(await fs.pathExists(basePath))) return [];
let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = await recFindByExt(basePath, ext);
var located_files = await exports.recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) {
let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
@@ -41,33 +42,33 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = await getJSONByType(type, id, basePath);
var jsonobj = await exports.getJSONByType(type, id, basePath);
if (!jsonobj) continue;
if (full_metadata) {
jsonobj['id'] = id;
files.push(jsonobj);
continue;
}
var upload_date = formatDateString(jsonobj.upload_date);
var upload_date = exports.formatDateString(jsonobj.upload_date);
var isaudio = type === 'audio';
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
var file_obj = new exports.File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
files.push(file_obj);
}
return files;
}
async function createContainerZipFile(file_name, container_file_objs) {
exports.createContainerZipFile = async (file_name, container_file_objs) => {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
return await exports.createZipFile(path.join('appdata', file_name + '.zip'), container_files_to_download);
}
async function createZipFile(zip_file_path, file_paths) {
exports.createZipFile = async (zip_file_path, file_paths) => {
let output = fs.createWriteStream(zip_file_path);
var archive = archiver('zip', {
@@ -95,7 +96,7 @@ async function createZipFile(zip_file_path, file_paths) {
return zip_file_path;
}
function getJSONMp4(name, customPath, openReadPerms = false) {
exports.getJSONMp4 = (name, customPath, openReadPerms = false) => {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
var jsonPath = path.join(customPath, name + ".info.json");
@@ -110,7 +111,7 @@ function getJSONMp4(name, customPath, openReadPerms = false) {
return obj;
}
function getJSONMp3(name, customPath, openReadPerms = false) {
exports.getJSONMp3 = (name, customPath, openReadPerms = false) => {
var obj = null;
if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
var jsonPath = path.join(customPath, name + ".info.json");
@@ -127,11 +128,11 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj;
}
function getJSON(file_path, type) {
exports.getJSON = (file_path, type) => {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
var jsonPath = exports.removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = exports.removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -142,12 +143,12 @@ function getJSON(file_path, type) {
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
exports.getJSONByType = (type, name, customPath, openReadPerms = false) => {
return type === 'audio' ? exports.getJSONMp3(name, customPath, openReadPerms) : exports.getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(file_path) {
const file_path_no_extension = removeFileExtension(file_path);
exports.getDownloadedThumbnail = (file_path) => {
const file_path_no_extension = exports.removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
@@ -163,7 +164,7 @@ function getDownloadedThumbnail(file_path) {
return null;
}
function getExpectedFileSize(input_info_jsons) {
exports.getExpectedFileSize = (input_info_jsons) => {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
@@ -186,12 +187,12 @@ function getExpectedFileSize(input_info_jsons) {
return expected_filesize;
}
function fixVideoMetadataPerms(file_path, type) {
exports.fixVideoMetadataPerms = (file_path, type) => {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const file_path_no_extension = exports.removeFileExtension(file_path);
const files_to_fix = [
// JSONs
@@ -208,10 +209,10 @@ function fixVideoMetadataPerms(file_path, type) {
}
}
function deleteJSONFile(file_path, type) {
exports.deleteJSONFile = (file_path, type) => {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const file_path_no_extension = exports.removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
@@ -220,58 +221,7 @@ function deleteJSONFile(file_path, type) {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
// archive helper functions
async function removeIDFromArchive(archive_path, type, id) {
const archive_file = path.join(archive_path, `archive_${type}.txt`);
const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
if (lastIndex === -1) return null;
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_file, updatedData);
if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
}
async function writeToBlacklist(archive_folder, type, line) {
let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
// adds newline to the beginning of the line
line.replace('\n', '');
line.replace('\r', '');
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
const archive_file = path.join(archive_path, `archive_${type}.txt`);
if (await fs.pathExists(archive_path)) {
const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
} else {
logger.info(`Could not find archive file for file ${uid}. Creating...`);
await fs.close(await fs.open(archive_file, 'w'));
}
}
function durationStringToNumber(dur_str) {
exports.durationStringToNumber = (dur_str) => {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
@@ -281,23 +231,22 @@ function durationStringToNumber(dur_str) {
return num_sum;
}
function getMatchingCategoryFiles(category, files) {
exports.getMatchingCategoryFiles = (category, files) => {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
exports.addUIDsToCategory = (category, files) => {
const files_that_match = exports.getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
function getCurrentDownloader() {
exports.getCurrentDownloader = () => {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
return details_json['downloader'];
}
async function recFindByExt(base, ext, files, result, recursive = true)
{
exports.recFindByExt = async (base, ext, files, result, recursive = true) => {
files = files || (await fs.readdir(base))
result = result || []
@@ -306,7 +255,7 @@ async function recFindByExt(base, ext, files, result, recursive = true)
if ( (await fs.stat(newbase)).isDirectory() )
{
if (!recursive) continue;
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
result = await exports.recFindByExt(newbase,ext,await fs.readdir(newbase),result)
}
else
{
@@ -319,17 +268,17 @@ async function recFindByExt(base, ext, files, result, recursive = true)
return result
}
function removeFileExtension(filename) {
exports.removeFileExtension = (filename) => {
const filename_parts = filename.split('.');
filename_parts.splice(filename_parts.length - 1);
return filename_parts.join('.');
}
function formatDateString(date_string) {
exports.formatDateString = (date_string) => {
return date_string ? `${date_string.substring(0, 4)}-${date_string.substring(4, 6)}-${date_string.substring(6, 8)}` : 'N/A';
}
function createEdgeNGrams(str) {
exports.createEdgeNGrams = (str) => {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
@@ -351,7 +300,7 @@ function createEdgeNGrams(str) {
// ffmpeg helper functions
async function cropFile(file_path, start, end, ext) {
exports.cropFile = async (file_path, start, end, ext) => {
return new Promise(resolve => {
const temp_file_path = `${file_path}.cropped${ext}`;
let base_ffmpeg_call = ffmpeg(file_path);
@@ -380,13 +329,13 @@ async function cropFile(file_path, start, end, ext) {
* setTimeout, but its a promise.
* @param {number} ms
*/
async function wait(ms) {
exports.wait = async (ms) => {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
async function checkExistsWithTimeout(filePath, timeout) {
exports.checkExistsWithTimeout = async (filePath, timeout) => {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
@@ -415,7 +364,7 @@ async function checkExistsWithTimeout(filePath, timeout) {
}
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
exports.fetchFile = async (url, path, file_label) => {
var len = null;
const res = await fetch(url);
@@ -442,7 +391,7 @@ async function fetchFile(url, path, file_label) {
});
}
async function restartServer(is_update = false) {
exports.restartServer = async (is_update = false) => {
logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
// the following line restarts the server through pm2
@@ -455,7 +404,7 @@ async function restartServer(is_update = false) {
// - if already exists and doesn't have value, ignore
// - if it doesn't exist and has value, add both arg and value
// - if it doesn't exist and doesn't have value, add arg
function injectArgs(original_args, new_args) {
exports.injectArgs = (original_args, new_args) => {
const updated_args = original_args.slice();
try {
for (let i = 0; i < new_args.length; i++) {
@@ -483,11 +432,11 @@ function injectArgs(original_args, new_args) {
return updated_args;
}
function filterArgs(args, args_to_remove) {
exports.filterArgs = (args, args_to_remove) => {
return args.filter(x => !args_to_remove.includes(x));
}
const searchObjectByString = function(o, s) {
exports.searchObjectByString = (o, s) => {
s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
s = s.replace(/^\./, ''); // strip a leading dot
var a = s.split('.');
@@ -502,7 +451,7 @@ const searchObjectByString = function(o, s) {
return o;
}
function stripPropertiesFromObject(obj, properties, whitelist = false) {
exports.stripPropertiesFromObject = (obj, properties, whitelist = false) => {
if (!whitelist) {
const new_obj = JSON.parse(JSON.stringify(obj));
for (let field of properties) {
@@ -518,7 +467,7 @@ function stripPropertiesFromObject(obj, properties, whitelist = false) {
return new_obj;
}
function getArchiveFolder(type, user_uid = null, sub = null) {
exports.getArchiveFolder = (type, user_uid = null, sub = null) => {
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
const subsFolderPath = config_api.getConfigItem('ytdl_subscriptions_base_path');
@@ -537,6 +486,38 @@ function getArchiveFolder(type, user_uid = null, sub = null) {
}
}
exports.getBaseURL = () => {
return `${config_api.getConfigItem('ytdl_url')}:${config_api.getConfigItem('ytdl_port')}`
}
exports.updateLoggerLevel = (new_logger_level) => {
const possible_levels = ['error', 'warn', 'info', 'verbose', 'debug'];
if (!possible_levels.includes(new_logger_level)) {
logger.error(`${new_logger_level} is not a valid logger level! Choose one of the following: ${possible_levels.join(', ')}.`)
new_logger_level = 'info';
}
logger.level = new_logger_level;
winston.loggers.get('console').level = new_logger_level;
logger.transports[2].level = new_logger_level;
}
exports.convertFlatObjectToNestedObject = (obj) => {
const result = {};
for (const key in obj) {
const nestedKeys = key.split('.');
let currentObj = result;
for (let i = 0; i < nestedKeys.length; i++) {
if (i === nestedKeys.length - 1) {
currentObj[nestedKeys[i]] = obj[key];
} else {
currentObj[nestedKeys[i]] = currentObj[nestedKeys[i]] || {};
currentObj = currentObj[nestedKeys[i]];
}
}
}
return result;
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -554,38 +535,7 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
this.view_count = view_count;
this.height = height;
this.abr = abr;
}
this.favorite = false;
}
exports.File = File;
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive: removeIDFromArchive,
writeToBlacklist: writeToBlacklist,
deleteFileFromArchive: deleteFileFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
formatDateString: formatDateString,
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
checkExistsWithTimeout: checkExistsWithTimeout,
fetchFile: fetchFile,
restartServer: restartServer,
injectArgs: injectArgs,
filterArgs: filterArgs,
searchObjectByString: searchObjectByString,
stripPropertiesFromObject: stripPropertiesFromObject,
getArchiveFolder: getArchiveFolder,
File: File
}