TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-08 23:31:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into add-yt-dlp

Browse Source
This commit is contained in:
Isaac Abadi
2021-07-20 21:55:18 -06:00
parent d11f77a6c9 380475b33e
commit 7aad7b7d24
74 changed files with 7026 additions and 2310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1058
backend/app.js
View File

File diff suppressed because it is too large Load Diff

7
backend/appdata/default.json
View File

@@ -20,7 +20,8 @@
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"enable_downloads_manager": true
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
@@ -53,6 +54,10 @@
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,

257
backend/authentication/auth.js
View File

@@ -1,12 +1,10 @@
const path = require('path');
const config_api = require('../config');
const consts = require('../consts');
var subscriptions_api = require('../subscriptions')
const fs = require('fs-extra');
var jwt = require('jsonwebtoken');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
var bcrypt = require('bcryptjs');
const bcrypt = require('bcryptjs');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -15,16 +13,15 @@ var JwtStrategy = require('passport-jwt').Strategy,
// other required vars
let logger = null;
let db = null;
let users_db = null;
let db_api = null;
let SERVER_SECRET = null;
let JWT_EXPIRATION = null;
let opts = null;
let saltRounds = null;
exports.initialize = function(input_db, input_users_db, input_logger) {
exports.initialize = function(db_api, input_logger) {
setLogger(input_logger)
setDB(input_db, input_users_db);
setDB(db_api);
/*************************
* Authentication module
@@ -34,21 +31,19 @@ exports.initialize = function(input_db, input_users_db, input_logger) {
JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
SERVER_SECRET = null;
if (users_db.get('jwt_secret').value()) {
SERVER_SECRET = users_db.get('jwt_secret').value();
if (db_api.users_db.get('jwt_secret').value()) {
SERVER_SECRET = db_api.users_db.get('jwt_secret').value();
} else {
SERVER_SECRET = uuid();
users_db.set('jwt_secret', SERVER_SECRET).write();
db_api.users_db.set('jwt_secret', SERVER_SECRET).write();
}
opts = {}
opts.jwtFromRequest = ExtractJwt.fromUrlQueryParameter('jwt');
opts.secretOrKey = SERVER_SECRET;
/*opts.issuer = 'example.com';
opts.audience = 'example.com';*/
exports.passport.use(new JwtStrategy(opts, function(jwt_payload, done) {
const user = users_db.get('users').find({uid: jwt_payload.user}).value();
exports.passport.use(new JwtStrategy(opts, async function(jwt_payload, done) {
const user = await db_api.getRecord('users', {uid: jwt_payload.user});
if (user) {
return done(null, user);
} else {
@@ -62,9 +57,8 @@ function setLogger(input_logger) {
logger = input_logger;
}
function setDB(input_db, input_users_db) {
db = input_db;
users_db = input_users_db;
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport');
@@ -80,7 +74,7 @@ exports.passport.deserializeUser(function(user, done) {
/***************************************
* Register user with hashed password
**************************************/
exports.registerUser = function(req, res) {
exports.registerUser = async function(req, res) {
var userid = req.body.userid;
var username = req.body.username;
var plaintextPassword = req.body.password;
@@ -98,20 +92,20 @@ exports.registerUser = function(req, res) {
}
bcrypt.hash(plaintextPassword, saltRounds)
.then(function(hash) {
.then(async function(hash) {
let new_user = generateUserObject(userid, username, hash);
// check if user exists
if (users_db.get('users').find({uid: userid}).value()) {
if (await db_api.getRecord('users', {uid: userid})) {
// user id is taken!
logger.error('Registration failed: UID is already taken!');
res.status(409).send('UID is already taken!');
} else if (users_db.get('users').find({name: username}).value()) {
} else if (await db_api.getRecord('users', {name: username})) {
// user name is taken!
logger.error('Registration failed: User name is already taken!');
res.status(409).send('User name is already taken!');
} else {
// add to db
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
logger.verbose(`New user created: ${new_user.name}`);
res.send({
user: new_user
@@ -144,16 +138,18 @@ exports.registerUser = function(req, res) {
************************************************/
exports.login = async (username, password) => {
const user = await db_api.getRecord('users', {name: username});
if (!user) { logger.error(`User ${username} not found`); false }
if (user.auth_method && user.auth_method !== 'internal') { return false }
return await bcrypt.compare(password, user.passhash) ? user : false;
}
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) {
return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
}
return done(null, await exports.login(username, password));
}
));
@@ -164,17 +160,17 @@ var getLDAPConfiguration = function(req, callback) {
};
exports.passport.use(new LdapStrategy(getLDAPConfiguration,
function(user, done) {
async function(user, done) {
// check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false);
const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value();
let db_user = await db_api.getRecord('users', {uid: user_uid});
if (!db_user) {
// generate DB user
let new_user = generateUserObject(user_uid, user_uid, null, 'ldap');
users_db.get('users').push(new_user).write();
await db_api.insertRecordIntoTable('users', new_user);
db_user = new_user;
logger.verbose(`Generated new user ${user_uid} using LDAP`);
}
@@ -198,11 +194,11 @@ exports.generateJWT = function(req, res, next) {
next();
}
exports.returnAuthResponse = function(req, res) {
exports.returnAuthResponse = async function(req, res) {
res.status(200).json({
user: req.user,
token: req.token,
permissions: exports.userPermissions(req.user.uid),
permissions: await exports.userPermissions(req.user.uid),
available_permissions: consts['AVAILABLE_PERMISSIONS']
});
}
@@ -215,7 +211,7 @@ exports.returnAuthResponse = function(req, res) {
* It also passes the user object to the next
* middleware through res.locals
**************************************/
exports.ensureAuthenticatedElseError = function(req, res, next) {
exports.ensureAuthenticatedElseError = (req, res, next) => {
var token = getToken(req.query);
if( token ) {
try {
@@ -233,10 +229,10 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
}
// change password
exports.changeUserPassword = async function(user_uid, new_pass) {
exports.changeUserPassword = async (user_uid, new_pass) => {
try {
const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
await db_api.updateRecord('users', {uid: user_uid}, {passhash: hash});
return true;
} catch (err) {
return false;
@@ -244,16 +240,15 @@ exports.changeUserPassword = async function(user_uid, new_pass) {
}
// change user permissions
exports.changeUserPermissions = function(user_uid, permission, new_value) {
exports.changeUserPermissions = async (user_uid, permission, new_value) => {
try {
const user_db_obj = users_db.get('users').find({uid: user_uid});
user_db_obj.get('permissions').pull(permission).write();
user_db_obj.get('permission_overrides').pull(permission).write();
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pullFromRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
if (new_value === 'yes') {
user_db_obj.get('permissions').push(permission).write();
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permissions', permission);
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
} else if (new_value === 'no') {
user_db_obj.get('permission_overrides').push(permission).write();
await db_api.pushToRecordsArray('users', {uid: user_uid}, 'permission_overrides', permission);
}
return true;
} catch (err) {
@@ -263,12 +258,11 @@ exports.changeUserPermissions = function(user_uid, permission, new_value) {
}
// change role permissions
exports.changeRolePermissions = function(role, permission, new_value) {
exports.changeRolePermissions = async (role, permission, new_value) => {
try {
const role_db_obj = users_db.get('roles').get(role);
role_db_obj.get('permissions').pull(permission).write();
await db_api.pullFromRecordsArray('roles', {key: role}, 'permissions', permission);
if (new_value === 'yes') {
role_db_obj.get('permissions').push(permission).write();
await db_api.pushToRecordsArray('roles', {key: role}, 'permissions', permission);
}
return true;
} catch (err) {
@@ -277,19 +271,19 @@ exports.changeRolePermissions = function(role, permission, new_value) {
}
}
exports.adminExists = function() {
return !!users_db.get('users').find({uid: 'admin'}).value();
exports.adminExists = async function() {
return !!(await db_api.getRecord('users', {uid: 'admin'}));
}
// video stuff
exports.getUserVideos = function(user_uid, type) {
const user = users_db.get('users').find({uid: user_uid}).value();
return type ? user['files'].filter(file => file.isAudio === (type === 'audio')) : user['files'];
exports.getUserVideos = async function(user_uid, type) {
const files = await db_api.getRecords('files', {user_uid: user_uid});
return type ? files.filter(file => file.isAudio === (type === 'audio')) : files;
}
exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
let file = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) {
let file = await db_api.getRecord('files', {file_uid: file_uid});
// prevent unauthorized users from accessing the file info
if (file && !file['sharingEnabled'] && requireSharing) file = null;
@@ -297,58 +291,22 @@ exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
return file;
}
exports.addPlaylist = function(user_uid, new_playlist) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
return true;
}
exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
return true;
}
exports.removePlaylist = function(user_uid, playlistID) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).remove({id: playlistID}).write();
exports.removePlaylist = async function(user_uid, playlistID) {
await db_api.removeRecord('playlist', {playlistID: playlistID});
return true;
}
exports.getUserPlaylists = function(user_uid, user_files = null) {
const user = users_db.get('users').find({uid: user_uid}).value();
const playlists = JSON.parse(JSON.stringify(user['playlists']));
const categories = db.get('categories').value();
if (categories && user_files) {
categories.forEach(category => {
const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio);
const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio);
if (audio_files && audio_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: audio_files[0].thumbnailURL,
thumbnailPath: audio_files[0].thumbnailPath,
fileNames: audio_files.map(file => file.id),
type: 'audio',
uid: user_uid,
auto: true
});
}
if (video_files && video_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: video_files[0].thumbnailURL,
thumbnailPath: video_files[0].thumbnailPath,
fileNames: video_files.map(file => file.id),
type: 'video',
uid: user_uid,
auto: true
});
}
});
}
return playlists;
exports.getUserPlaylists = async function(user_uid, user_files = null) {
return await db_api.getRecords('playlists', {user_uid: user_uid});
}
exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false) {
let playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).value();
exports.getUserPlaylist = async function(user_uid, playlistID, requireSharing = false) {
let playlist = await db_api.getRecord('playlists', {id: playlistID});
// prevent unauthorized users from accessing the file info
if (requireSharing && !playlist['sharingEnabled']) playlist = null;
@@ -356,109 +314,23 @@ exports.getUserPlaylist = function(user_uid, playlistID, requireSharing = false)
return playlist;
}
exports.registerUserFile = function(user_uid, file_object) {
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
path: file_object['path']
}).write();
users_db.get('users').find({uid: user_uid}).get(`files`)
.push(file_object)
.write();
}
exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) {
exports.changeSharingMode = async function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
if (file_obj) {
const type = file_obj.isAudio ? 'audio' : 'video';
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
// close descriptors
if (config_api.descriptors[file_obj.id]) {
try {
for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) {
config_api.descriptors[file_obj.id][i].destroy();
}
} catch(e) {
}
}
const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext);
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
uid: file_uid
}).write();
if (await fs.pathExists(full_path)) {
// remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null;
if (await fs.pathExists(json_path)) {
youtube_id = await fs.readJSON(json_path).id;
await fs.unlink(json_path);
} else if (await fs.pathExists(alternate_json_path)) {
youtube_id = await fs.readJSON(alternate_json_path).id;
await fs.unlink(alternate_json_path);
}
await fs.unlink(full_path);
// do archive stuff
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
} else {
logger.info(`Could not find archive file for ${type} files. Creating...`);
await fs.ensureFile(archive_path);
}
}
}
success = true;
} else {
success = false;
logger.warn(`User file ${file_uid} does not exist!`);
}
is_playlist ? await db_api.updateRecord(`playlists`, {id: file_uid}, {sharingEnabled: enabled}) : await db_api.updateRecord(`files`, {uid: file_uid}, {sharingEnabled: enabled});
success = true;
return success;
}
exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid});
if (user_db_obj.value()) {
const file_db_obj = is_playlist ? user_db_obj.get(`playlists`).find({id: file_uid}) : user_db_obj.get(`files`).find({uid: file_uid});
if (file_db_obj.value()) {
success = true;
file_db_obj.assign({sharingEnabled: enabled}).write();
}
}
exports.userHasPermission = async function(user_uid, permission) {
return success;
}
exports.userHasPermission = function(user_uid, permission) {
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return false;
}
const role_permissions = (users_db.get('roles').value())['permissions'];
const role_permissions = (await db_api.getRecords('roles'))['permissions'];
const user_has_explicit_permission = user_obj['permissions'].includes(permission);
const permission_in_overrides = user_obj['permission_overrides'].includes(permission);
@@ -481,16 +353,17 @@ exports.userHasPermission = function(user_uid, permission) {
}
}
exports.userPermissions = function(user_uid) {
exports.userPermissions = async function(user_uid) {
let user_permissions = [];
const user_obj = users_db.get('users').find({uid: user_uid}).value();
const user_obj = await db_api.getRecord('users', ({uid: user_uid}));
const role = user_obj['role'];
if (!role) {
// role doesn't exist
logger.error('Invalid role ' + role);
return null;
}
const role_permissions = users_db.get('roles').get(role).get('permissions').value()
const role_obj = await db_api.getRecord('roles', {key: role});
const role_permissions = role_obj['permissions'];
for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) {
let permission = consts['AVAILABLE_PERMISSIONS'][i];

62
backend/categories.js
View File

@@ -1,4 +1,5 @@
const config_api = require('./config');
const utils = require('./utils');
var logger = null;
var db = null;
@@ -33,35 +34,58 @@ Rules:
*/
async function categorize(file_json) {
async function categorize(file_jsons) {
// to make the logic easier, let's assume the file metadata is an array
if (!Array.isArray(file_jsons)) file_jsons = [file_jsons];
let selected_category = null;
const categories = getCategories();
const categories = await getCategories();
if (!categories) {
logger.warn('Categories could not be found. Initializing categories...');
db.assign({categories: []}).write();
logger.warn('Categories could not be found.');
return null;
return;
}
for (let i = 0; i < categories.length; i++) {
const category = categories[i];
const rules = category['rules'];
// if rules for current category apply, then that is the selected category
if (applyCategoryRules(file_json, rules, category['name'])) {
selected_category = category;
logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`);
return selected_category;
for (let i = 0; i < file_jsons.length; i++) {
const file_json = file_jsons[i];
for (let j = 0; j < categories.length; j++) {
const category = categories[j];
const rules = category['rules'];
// if rules for current category apply, then that is the selected category
if (applyCategoryRules(file_json, rules, category['name'])) {
selected_category = category;
logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`);
return selected_category;
}
}
}
return selected_category;
}
function getCategories() {
const categories = db.get('categories').value();
async function getCategories() {
const categories = await db_api.getRecords('categories');
return categories ? categories : null;
}
async function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = await getCategories();
if (available_categories && files) {
for (category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
category['thumbnailPath'] = files_that_match[0].thumbnailPath;
category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
category['id'] = category['uid'];
categories_as_playlists.push(category);
}
}
}
return categories_as_playlists;
}
function applyCategoryRules(file_json, rules, category_name) {
let rules_apply = false;
for (let i = 0; i < rules.length; i++) {
@@ -72,10 +96,10 @@ function applyCategoryRules(file_json, rules, category_name) {
switch (rule['comparator']) {
case 'includes':
rule_applies = file_json[rule['property']].includes(rule['value']);
rule_applies = file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase());
break;
case 'not_includes':
rule_applies = !(file_json[rule['property']].includes(rule['value']));
rule_applies = !(file_json[rule['property']].toLowerCase().includes(rule['value'].toLowerCase()));
break;
case 'equals':
rule_applies = file_json[rule['property']] === rule['value'];
@@ -120,4 +144,6 @@ async function addTagToExistingTags(tag) {
module.exports = {
initialize: initialize,
categorize: categorize,
getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists
}

7
backend/config.js
View File

@@ -197,7 +197,8 @@ DEFAULT_CONFIG = {
"allow_quality_select": true,
"download_only_mode": false,
"allow_multi_download_mode": true,
"enable_downloads_manager": true
"enable_downloads_manager": true,
"allow_playlist_categorization": true
},
"API": {
"use_API_key": false,
@@ -230,6 +231,10 @@ DEFAULT_CONFIG = {
"searchFilter": "(uid={{username}})"
}
},
"Database": {
"use_local_db": false,
"mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
},
"Advanced": {
"default_downloader": "youtube-dl",
"use_default_downloading_agent": true,

14
backend/consts.js
View File

@@ -68,6 +68,10 @@ let CONFIG_ITEMS = {
'key': 'ytdl_enable_downloads_manager',
'path': 'YoutubeDLMaterial.Extra.enable_downloads_manager'
},
'ytdl_allow_playlist_categorization': {
'key': 'ytdl_allow_playlist_categorization',
'path': 'YoutubeDLMaterial.Extra.allow_playlist_categorization'
},
// API
'ytdl_use_api_key': {
@@ -149,6 +153,16 @@ let CONFIG_ITEMS = {
'path': 'YoutubeDLMaterial.Users.ldap_config'
},
// Database
'ytdl_use_local_db': {
'key': 'ytdl_use_local_db',
'path': 'YoutubeDLMaterial.Database.use_local_db'
},
'ytdl_mongodb_connection_string': {
'key': 'ytdl_mongodb_connection_string',
'path': 'YoutubeDLMaterial.Database.mongodb_connection_string'
},
// Advanced
'ytdl_default_downloader': {
'key': 'ytdl_default_downloader',

923
backend/db.js
View File

File diff suppressed because it is too large Load Diff

2
backend/entrypoint.sh
View File

@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
CMD="node app.js"
CMD="forever app.js"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then

1260
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

10
backend/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon -q app.js"
"start": "nodemon app.js",
"debug": "set YTDL_MODE=debug && node app.js"
},
"nodemonConfig": {
"ignore": [
@@ -14,7 +15,8 @@
"public/*"
],
"watch": [
"restart.json"
"restart_update.json",
"restart_general.json"
]
},
"repository": {
@@ -43,11 +45,13 @@
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"mocha": "^8.4.0",
"moment": "^2.29.1",
"mongodb": "^3.6.9",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"node-id3": "^0.1.14",
"nodemon": "^2.0.2",
"nodemon": "^2.0.7",
"passport": "^0.4.1",
"passport-http": "^0.3.0",
"passport-jwt": "^4.0.0",

205
backend/subscriptions.js
View File

@@ -14,13 +14,13 @@ const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
let db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setDB(input_db_api) { db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
function initialize(input_db_api, input_logger) {
setDB(input_db_api);
setLogger(input_logger);
}
@@ -34,12 +34,7 @@ async function subscribe(sub, user_uid = null) {
sub.isPlaylist = sub.url.includes('playlist');
sub.videos = [];
let url_exists = false;
if (user_uid)
url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value()
else
url_exists = !!db.get('subscriptions').find({url: sub.url}).value();
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
if (!sub.name && url_exists) {
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
@@ -48,19 +43,12 @@ async function subscribe(sub, user_uid = null) {
return;
}
// add sub to db
let sub_db = null;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
sub['user_uid'] = user_uid ? user_uid : undefined;
await db_api.insertRecordIntoTable('subscriptions', sub);
let success = await getSubscriptionInfo(sub, user_uid);
if (success) {
sub = sub_db.value();
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
@@ -91,8 +79,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
return new Promise(async resolve => {
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id);
}
@@ -122,10 +110,7 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
// if it's now valid, update
if (sub.name) {
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
}
}
@@ -141,10 +126,8 @@ async function getSubscriptionInfo(sub, user_uid = null) {
// updates subscription
sub.archive = archive_dir;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
else
db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write();
await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
}
// TODO: get even more info
@@ -166,10 +149,8 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
let result_obj = { success: false, error: '' };
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
await db_api.removeRecord('subscriptions', {id: id});
await db_api.removeAllRecords('files', {sub_id: id});
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
@@ -191,20 +172,16 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
// TODO: combine this with deletefile
let basePath = null;
let sub_db = null;
if (user_uid) {
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
sub_db = db.get('subscriptions').find({id: sub.id});
}
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
await db_api.removeRecord('files', {uid: file_uid});
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
@@ -243,7 +220,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
utils.removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
@@ -255,14 +232,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
}
async function getVideosForSub(sub, user_uid = null) {
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
const latest_sub_obj = sub_db.value();
const latest_sub_obj = await getSubscription(sub.id);
if (!latest_sub_obj || latest_sub_obj['downloading']) {
return false;
}
@@ -277,6 +247,7 @@ async function getVideosForSub(sub, user_uid = null) {
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appendedBasePath = getAppendedBasePath(sub, basePath);
fs.ensureDirSync(appendedBasePath);
let multiUserMode = null;
if (user_uid) {
@@ -291,9 +262,18 @@ async function getVideosForSub(sub, user_uid = null) {
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) {
return new Promise(async resolve => {
const preimported_file_paths = [];
const PREIMPORT_INTERVAL = 5000;
const preregister_check = setInterval(async () => {
if (sub.streamingOnly) return;
await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
}, PREIMPORT_INTERVAL);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
@@ -303,7 +283,7 @@ async function getVideosForSub(sub, user_uid = null) {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
await handleOutputJSON(sub, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
@@ -337,7 +317,7 @@ async function getVideosForSub(sub, user_uid = null) {
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
}
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
@@ -351,6 +331,7 @@ async function getVideosForSub(sub, user_uid = null) {
}, err => {
logger.error(err);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
});
}
@@ -433,8 +414,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
return downloadConfig;
}
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) {
async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
// TODO: remove streaming only mode
if (false && sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
@@ -448,12 +430,15 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object);
if (sub_db.get('videos').find({path: path_string}).value()) {
const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id});
if (file_exists) {
// TODO: fix issue where files of different paths due to custom path get downloaded multiple times
// file already exists in DB, return early to avoid reseting the download date
return;
}
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);
const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
@@ -466,73 +451,41 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
}
}
function getSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();
else
return db.get('subscriptions').value();
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}
function getAllSubscriptions() {
let subscriptions = null;
async function getAllSubscriptions() {
const all_subs = await db_api.getRecords('subscriptions');
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (multiUserMode) {
subscriptions = [];
let users = users_db.get('users').value();
for (let i = 0; i < users.length; i++) {
if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']);
}
} else {
subscriptions = getSubscriptions();
}
return subscriptions;
return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
}
function getSubscription(subID, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return db.get('subscriptions').find({id: subID}).value();
async function getSubscription(subID) {
return await db_api.getRecord('subscriptions', {id: subID});
}
function getSubscriptionByName(subName, user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value();
else
return db.get('subscriptions').find({name: subName}).value();
async function getSubscriptionByName(subName, user_uid = null) {
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}
function updateSubscription(sub, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(sub).write();
}
async function updateSubscription(sub, user_uid = null) {
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
return true;
}
function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(sub => {
updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
subs.forEach(async sub => {
await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
});
}
function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
} else {
db.get('subscriptions').find({id: sub.id}).assign(assignment_obj).write();
}
async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
// TODO: combine with updateSubscription
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
return true;
}
function subExists(subID, user_uid = null) {
if (user_uid)
return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value();
else
return !!db.get('subscriptions').find({id: subID}).value();
}
async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
@@ -548,7 +501,7 @@ async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
checkVideoIfBetterExists(video, sub, user_uid)
await checkVideoIfBetterExists(video, sub, user_uid)
}
});
}
@@ -558,14 +511,14 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
if (err) {
// video is not available anymore for whatever reason
} else if (output) {
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
if (output[metric_to_compare] > file_obj[metric_to_compare]) {
// download new video as the simulated one is better
youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => {
youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
if (err) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (output) {
@@ -586,33 +539,6 @@ function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
@@ -623,7 +549,6 @@ module.exports = {
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
removeIDFromArchive : removeIDFromArchive,
setLogger : setLogger,
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple

290
backend/test/tests.js Normal file
View File

@@ -0,0 +1,290 @@
var assert = require('assert');
const low = require('lowdb')
var winston = require('winston');
process.chdir('./backend')
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync('./appdata/db.json');
const db = low(adapter)
const users_adapter = new FileSync('./appdata/users.json');
const users_db = low(users_adapter);
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
let debugMode = process.env.YTDL_MODE === 'debug';
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: 'debug', name: 'console'})
]
});
var auth_api = require('../authentication/auth');
var db_api = require('../db');
const utils = require('../utils');
const subscriptions_api = require('../subscriptions');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
db_api.initialize(db, users_db, logger);
describe('Database', async function() {
describe('Import', async function() {
it('Migrate', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords();
const success = await db_api.importJSONToDB(db.value(), users_db.value());
assert(success);
});
it('Transfer to remote', async function() {
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(true);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
it('Transfer to local', async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
await db_api.transferDB(false);
const success = await db_api.getRecord('test', {test: 'test'});
assert(success);
});
});
describe('Export', function() {
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
it('Bulk add', async function() {
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecords('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon: http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
assert(success);
});
});
});
describe('Multi User', async function() {
let user = null;
const user_to_test = 'admin';
const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const playlist_to_test = 'ysabVZz4x';
beforeEach(async function() {
await db_api.connectToDB();
auth_api.initialize(db_api, logger);
subscriptions_api.initialize(db_api, logger);
user = await auth_api.login('admin', 'pass');
});
describe('Authentication', function() {
it('login', async function() {
assert(user);
});
});
describe('Video player - normal', function() {
const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
it('Get video', async function() {
const video_obj = db_api.getVideo(video_to_test, 'admin');
assert(video_obj);
});
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
const playlist_files_to_download = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const playlist_file = await db_api.getVideo(uid, user_to_test);
playlist_files_to_download.push(playlist_file);
}
const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
it('Subscription zip generator', async function() {
const sub = await subscriptions_api.getSubscription(sub_to_test, user_to_test);
const sub_videos = await db_api.getRecords('files', {sub_id: sub.id});
assert(sub);
const sub_files_to_download = [];
for (let i = 0; i < sub_videos.length; i++) {
const sub_file = sub_videos[i];
sub_files_to_download.push(sub_file);
}
const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
const zip_exists = fs.pathExistsSync(zip_path);
assert(zip_exists);
if (zip_exists) fs.unlinkSync(zip_path);
});
});
// describe('Video player - subscription', function() {
// const sub_to_test = '';
// const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';
// it('Get video', async function() {
// const video_obj = db_api.getVideo(video_to_test, 'admin', );
// assert(video_obj);
// });
// it('Video access - disallowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(!video_obj);
// });
// it('Video access - allowed', async function() {
// await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test, sub_to_test);
// const video_obj = auth_api.getUserVideo('admin', video_to_test, true);
// assert(video_obj);
// });
// });
});

174
backend/utils.js
View File

@@ -1,6 +1,7 @@
var fs = require('fs-extra')
var path = require('path')
const fs = require('fs-extra')
const path = require('path')
const config_api = require('./config');
const archiver = require('archiver');
const is_windows = process.platform === 'win32';
@@ -52,6 +53,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
return files;
}
async function createContainerZipFile(container_obj, container_file_objs) {
const container_files_to_download = [];
for (let i = 0; i < container_file_objs.length; i++) {
const container_file_obj = container_file_objs[i];
container_files_to_download.push(container_file_obj.path);
}
return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
}
async function createZipFile(zip_file_path, file_paths) {
let output = fs.createWriteStream(zip_file_path);
var archive = archiver('zip', {
gzip: true,
zlib: { level: 9 } // Sets the compression level.
});
archive.on('error', function(err) {
logger.error(err);
throw err;
});
// pipe archive data to the output file
archive.pipe(output);
for (let file_path of file_paths) {
const file_name = path.parse(file_path).base;
archive.file(file_path, {name: file_name})
}
await archive.finalize();
// wait a tiny bit for the zip to reload in fs
await wait(100);
return zip_file_path;
}
function getJSONMp4(name, customPath, openReadPerms = false) {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -84,6 +122,21 @@ function getJSONMp3(name, customPath, openReadPerms = false) {
return obj;
}
function getJSON(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
let obj = null;
var jsonPath = removeFileExtension(file_path) + '.info.json';
var alternateJsonPath = removeFileExtension(file_path) + `${ext}.info.json`;
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
}
else obj = 0;
return obj;
}
function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
@@ -105,6 +158,23 @@ function getDownloadedThumbnail(name, type, customPath = null) {
return null;
}
function getDownloadedThumbnail2(file_path, type) {
const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg';
let webpPath = file_path_no_extension + '.webp';
let pngPath = file_path_no_extension + '.png';
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getExpectedFileSize(input_info_jsons) {
// treat single videos as arrays to have the file sizes checked/added to. makes the code cleaner
const info_jsons = Array.isArray(input_info_jsons) ? input_info_jsons : [input_info_jsons];
@@ -152,6 +222,28 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
}
}
function fixVideoMetadataPerms2(file_path, type) {
if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
const files_to_fix = [
// JSONs
file_path_no_extension + '.info.json',
file_path_no_extension + ext + '.info.json',
// Thumbnails
file_path_no_extension + '.webp',
file_path_no_extension + '.jpg'
];
for (const file of files_to_fix) {
if (!fs.existsSync(file)) continue;
fs.chmodSync(file, 0o644);
}
}
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
@@ -164,6 +256,64 @@ function deleteJSONFile(name, type, customPath = null) {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
function deleteJSONFile2(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path);
let json_path = file_path_no_extension + '.info.json';
let alternate_json_path = file_path_no_extension + ext + '.info.json';
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
function durationStringToNumber(dur_str) {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
for (let i = dur_str_parts.length-1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
}
return num_sum;
}
function getMatchingCategoryFiles(category, files) {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
async function recFindByExt(base,ext,files,result)
{
@@ -193,6 +343,16 @@ function removeFileExtension(filename) {
return filename_parts.join('.');
}
/**
* setTimeout, but its a promise.
* @param {number} ms
*/
async function wait(ms) {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -215,13 +375,23 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getJSON: getJSON,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getDownloadedThumbnail2: getDownloadedThumbnail2,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
fixVideoMetadataPerms2: fixVideoMetadataPerms2,
deleteJSONFile: deleteJSONFile,
deleteJSONFile2: deleteJSONFile2,
removeIDFromArchive, removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
wait: wait,
File: File
}