TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added new utils.js module to assist backend with shared helper functions

Browse Source 
Subscription files are now stored in the database, and will be primarily managed through it
This commit is contained in:
Tzahi12345
2020-06-09 18:02:25 -04:00
parent f04139634a
commit b8a4e0773f
4 changed files with 205 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
backend/app.js
View File

@@ -13,6 +13,8 @@ var express = require("express");
var bodyParser = require("body-parser");
var archiver = require('archiver');
var unzipper = require('unzipper');
var db_api = require('./db')
var utils = require('./utils')
var mergeFiles = require('merge-files');
const low = require('lowdb')
var ProgressBar = require('progress');
@@ -73,8 +75,9 @@ const logger = winston.createLogger({
});
config_api.initialize(logger);
subscriptions_api.initialize(db, users_db, logger);
auth_api.initialize(users_db, logger);
db_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api);
// var GithubContent = require('github-content');
@@ -191,27 +194,12 @@ app.use(bodyParser.json());
// use passport
app.use(auth_api.passport.initialize());
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) {
this.id = id;
this.title = title;
this.thumbnailURL = thumbnailURL;
this.isAudio = isAudio;
this.duration = duration;
this.url = url;
this.uploader = uploader;
this.size = size;
this.path = path;
this.upload_date = upload_date;
}
// actual functions
async function checkMigrations() {
return new Promise(async resolve => {
// 3.5->3.6 migration
const files_to_db_migration_complete = db.get('files_to_db_migration_complete').value();
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+')
@@ -236,7 +224,7 @@ async function runFilesToDBMigration() {
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
registerFileDB(file_obj.id + '.mp3', 'audio');
db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
}
@@ -245,7 +233,7 @@ async function runFilesToDBMigration() {
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
registerFileDB(file_obj.id + '.mp4', 'video');
db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
}
@@ -654,6 +642,11 @@ async function watchSubscriptions() {
continue;
}
if (!sub.name) {
logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
continue;
}
logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
setTimeout(async () => {
await subscriptions_api.getVideosForSub(sub, sub.user_uid);
@@ -700,7 +693,7 @@ function getMp3s() {
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONMp3(id);
var jsonobj = utils.getJSONMp3(id, audioFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@@ -713,7 +706,7 @@ function getMp3s() {
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = true;
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
mp3s.push(file_obj);
}
return mp3s;
@@ -729,7 +722,7 @@ function getMp4s(relative_path = true) {
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONMp4(id);
var jsonobj = utils.getJSONMp4(id, videoFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@@ -742,7 +735,7 @@ function getMp4s(relative_path = true) {
var size = stats.size;
var isaudio = false;
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
mp4s.push(file_obj);
}
return mp4s;
@@ -750,14 +743,14 @@ function getMp4s(relative_path = true) {
function getThumbnailMp3(name)
{
var obj = getJSONMp3(name);
var obj = utils.getJSONMp3(name, audioFolderPath);
var thumbnailLink = obj.thumbnail;
return thumbnailLink;
}
function getThumbnailMp4(name)
{
var obj = getJSONMp4(name);
var obj = utils.getJSONMp4(name, videoFolderPath);
var thumbnailLink = obj.thumbnail;
return thumbnailLink;
}
@@ -794,54 +787,6 @@ function getFileSizeMp4(name)
return filesize;
}
function getJSONMp3(name, customPath = null, openReadPerms = false)
{
var jsonPath = audioFolderPath+name+".info.json";
var alternateJsonPath = audioFolderPath+name+".mp3.info.json";
if (!customPath) {
jsonPath = audioFolderPath + name + ".info.json";
} else {
jsonPath = customPath + name + ".info.json";
alternateJsonPath = customPath + name + ".mp3.info.json";
}
var obj = null;
if (fs.existsSync(jsonPath)) {
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
if (!is_windows && openReadPerms) fs.chmodSync(jsonPath, 0o755);
}
else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
if (!is_windows && openReadPerms) fs.chmodSync(alternateJsonPath, 0o755);
}
else
obj = 0;
return obj;
}
function getJSONMp4(name, customPath = null, openReadPerms = false)
{
var obj = null; // output
let jsonPath = null;
var alternateJsonPath = videoFolderPath + name + ".mp4.info.json";
if (!customPath) {
jsonPath = videoFolderPath + name + ".info.json";
} else {
jsonPath = customPath + name + ".info.json";
alternateJsonPath = customPath + name + ".mp4.info.json";
}
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
if (openReadPerms) fs.chmodSync(jsonPath, 0o644);
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
if (openReadPerms) fs.chmodSync(alternateJsonPath, 0o644);
}
else obj = 0;
return obj;
}
function getAmountDownloadedMp3(name)
{
var partPath = audioFolderPath+name+".mp3.part";
@@ -965,7 +910,7 @@ async function deleteAudioFile(name, blacklistMode = false) {
// get ID from JSON
var jsonobj = getJSONMp3(name);
var jsonobj = utils.getJSONMp3(name, audioFolderPath);
let id = null;
if (jsonobj) id = jsonobj.id;
@@ -1023,7 +968,7 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON
var jsonobj = getJSONMp4(name);
var jsonobj = utils.getJSONMp4(name, videoFolderPath);
let id = null;
if (jsonobj) id = jsonobj.id;
@@ -1077,7 +1022,7 @@ function recFindByExt(base,ext,files,result)
)
return result
}
/*
function registerFileDB(file_path, type, multiUserMode = null) {
const file_id = file_path.substring(0, file_path.length-4);
const file_object = generateFileObject(file_id, type, multiUserMode && multiUserMode.file_path);
@@ -1112,7 +1057,7 @@ function registerFileDB(file_path, type, multiUserMode = null) {
}
function generateFileObject(id, type, customPath = null) {
var jsonobj = (type === 'audio') ? getJSONMp3(id, customPath, true) : getJSONMp4(id, customPath, true);
var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
if (!jsonobj) {
return null;
}
@@ -1132,10 +1077,10 @@ function generateFileObject(id, type, customPath = null) {
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date);
return file_obj;
}
*/
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
@@ -1292,7 +1237,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
}
// registers file in DB
file_uid = registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode);
file_uid = db_api.registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode);
if (file_name) file_names.push(file_name);
}
@@ -1431,7 +1376,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) {
// registers file in DB
const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length);
file_uid = registerFileDB(base_file_name, type, multiUserMode);
file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode);
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
@@ -2234,45 +2179,48 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
// get sub videos
if (subscription.name && !subscription.streamingOnly) {
let base_path = null;
if (user_uid)
base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
var parsed_files = subscription.videos;
if (!parsed_files) {
let base_path = null;
if (user_uid)
base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = recFindByExt(appended_base_path, 'mp4');
} catch(e) {
files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);
res.sendStatus(500);
return;
}
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(appended_base_path.length, file.length);
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp4(id, appended_base_path);
if (!jsonobj) continue;
var title = jsonobj.title;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
var size = stats.size;
var isaudio = false;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
parsed_files.push(file_obj);
}
}
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = recFindByExt(appended_base_path, 'mp4');
} catch(e) {
files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);
res.sendStatus(500);
return;
}
var parsed_files = [];
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(appended_base_path.length, file.length);
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONMp4(id, appended_base_path);
if (!jsonobj) continue;
var title = jsonobj.title;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
var size = stats.size;
var isaudio = false;
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
parsed_files.push(file_obj);
}
res.send({
subscription: subscription,
@@ -2284,7 +2232,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
if (subscription.videos) {
for (let i = 0; i < subscription.videos.length; i++) {
const video = subscription.videos[i];
parsed_files.push(new File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date));
parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date));
}
}
res.send({

48
backend/db.js
View File

@@ -1,3 +1,8 @@
var fs = require('fs-extra')
var path = require('path')
var utils = require('./utils')
const { uuid } = require('uuidv4');
var logger = null;
var db = null;
var users_db = null;
@@ -23,8 +28,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
if (multiUserMode) {
if (!sub) {
if (!sub) {
if (multiUserMode) {
const user_uid = multiUserMode.user;
users_db.get('users').find({uid: user_uid}).get(`files.${type}`)
.remove({
@@ -35,36 +40,37 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
.push(file_object)
.write();
} else {
}
} else if (!sub) {
// remove existing video if overwriting
db.get(`files.${type}`)
.remove({
path: file_object['path']
}).write();
// remove existing video if overwriting
db.get(`files.${type}`)
.remove({
path: file_object['path']
}).write();
db.get(`files.${type}`)
.push(file_object)
.write();
} else if (sub) {
if (multi)
db.get(`files.${type}`)
.push(file_object)
.write();
}
} else {
// this should never be used
logger.error('Failed to determine file type during video DB registration.');
return null;
sub_db = null;
if (multiUserMode) {
const user_uid = multiUserMode.user;
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
sub_db = db.get('subscriptions').find({id: sub.id});
}
sub_db.get('videos').push(file_object).write();
}
return file_object['uid'];
}
function generateFileObject(id, type, customPath = null) {
var jsonobj = (type === 'audio') ? getJSONMp3(id, customPath, true) : getJSONMp4(id, customPath, true);
var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
if (!jsonobj) {
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const file_path = getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
// console.
var stats = fs.statSync(path.join(__dirname, file_path));
@@ -79,7 +85,7 @@ function generateFileObject(id, type, customPath = null) {
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date);
return file_obj;
}

49
backend/subscriptions.js
View File

@@ -6,17 +6,20 @@ var path = require('path');
var youtubedl = require('youtube-dl');
const config_api = require('./config');
var utils = require('./utils')
const debugMode = process.env.YTDL_MODE === 'debug';
var logger = null;
var db = null;
var users_db = null;
function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db }
var db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger) {
setDB(input_db, input_users_db);
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
}
@@ -28,6 +31,8 @@ async function subscribe(sub, user_uid = null) {
return new Promise(async resolve => {
// sub should just have url and name. here we will get isPlaylist and path
sub.isPlaylist = sub.url.includes('playlist');
sub.type = 'video'; // TODO: eventually change
sub.videos = [];
let url_exists = false;
@@ -44,15 +49,25 @@ async function subscribe(sub, user_uid = null) {
}
// add sub to db
if (user_uid)
let sub_db = null;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write();
else
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
} else {
db.get('subscriptions').push(sub).write();
sub_db = db.get('subscriptions').find({id: sub.id});
}
let success = await getSubscriptionInfo(sub, user_uid);
if (success) {
sub = sub_db.get().value();
getVideosForSub(sub, user_uid);
} else {
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
};
result_obj.success = success;
result_obj.sub = sub;
getVideosForSub(sub, user_uid);
resolve(result_obj);
});
@@ -237,10 +252,14 @@ async function getVideosForSub(sub, user_uid = null) {
const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive');
let appendedBasePath = null
if (sub.name) {
appendedBasePath = getAppendedBasePath(sub, basePath);
} else {
appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'));
appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json'];
@@ -286,7 +305,7 @@ async function getVideosForSub(sub, user_uid = null) {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0)
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
fs.appendFileSync(archive_path, output['id']);
@@ -315,7 +334,7 @@ async function getVideosForSub(sub, user_uid = null) {
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, reset_videos);
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
// TODO: Potentially store downloaded files in db?
@@ -328,7 +347,7 @@ async function getVideosForSub(sub, user_uid = null) {
});
}
function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) {
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
if (sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
@@ -341,7 +360,7 @@ function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) {
sub_db.get('videos').push(output_json).write();
} else {
// TODO: make multiUserMode obj
db_api.registerFileDB(output_json['_filename'], sub.type, multiUserMode, sub);
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
}
}

76
backend/utils.js Normal file
View File

@@ -0,0 +1,76 @@
var fs = require('fs-extra')
var path = require('path')
const config_api = require('./config');
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
let unfixed_parts = unfixed_path.split('.');
const old_ext = unfixed_parts[unfixed_parts.length-1];
if (old_ext !== new_ext) {
unfixed_parts[unfixed_parts.length-1] = new_ext;
fixed_path = unfixed_parts.join('.');
}
return fixed_path;
}
function getJSONMp4(name, customPath, openReadPerms = false) {
var obj = null; // output
if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
var jsonPath = path.join(customPath, name + ".info.json");
var alternateJsonPath = path.join(customPath, name + ".mp4.info.json");
if (fs.existsSync(jsonPath))
{
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
if (openReadPerms) fs.chmodSync(jsonPath, 0o644);
} else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
if (openReadPerms) fs.chmodSync(alternateJsonPath, 0o644);
}
else obj = 0;
return obj;
}
function getJSONMp3(name, customPath, openReadPerms = false) {
var obj = null;
if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path');
var jsonPath = customPath + name + ".info.json";
var alternateJsonPath = customPath + name + ".mp3.info.json";
if (fs.existsSync(jsonPath)) {
obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
if (!is_windows && openReadPerms) fs.chmodSync(jsonPath, 0o755);
}
else if (fs.existsSync(alternateJsonPath)) {
obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8'));
if (!is_windows && openReadPerms) fs.chmodSync(alternateJsonPath, 0o755);
}
else
obj = 0;
return obj;
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) {
this.id = id;
this.title = title;
this.thumbnailURL = thumbnailURL;
this.isAudio = isAudio;
this.duration = duration;
this.url = url;
this.uploader = uploader;
this.size = size;
this.path = path;
this.upload_date = upload_date;
}
module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getTrueFileName: getTrueFileName,
File: File
}