TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-04 12:21:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

added better logging in app.js using winston

Browse Source
This commit is contained in:
Isaac Grynsztein
2020-04-02 23:05:17 -04:00
parent 717f6deb11
commit 2f0bbca15c
2 changed files with 77 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
backend/app.js
View File

@@ -1,6 +1,7 @@
var async = require('async'); var async = require('async');
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
var fs = require('fs-extra'); var fs = require('fs-extra');
var winston = require('winston');
var path = require('path'); var path = require('path');
var youtubedl = require('youtube-dl'); var youtubedl = require('youtube-dl');
var compression = require('compression'); var compression = require('compression');
@@ -26,10 +27,32 @@ const { spawn } = require('child_process')
var app = express(); var app = express();
// database setup
const FileSync = require('lowdb/adapters/FileSync') const FileSync = require('lowdb/adapters/FileSync')
const adapter = new FileSync('./appdata/db.json'); const adapter = new FileSync('./appdata/db.json');
const db = low(adapter) const db = low(adapter)
// logging setup
// console format
const defaultFormat = winston.format.printf(({ level, message, label, timestamp }) => {
return `${timestamp} ${level.toUpperCase()}: ${message}`;
});
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.timestamp(), defaultFormat),
defaultMeta: {},
transports: [
//
// - Write to all logs with level `info` and below to `combined.log`
// - Write all logs error (and below) to `error.log`.
//
new winston.transports.File({ filename: 'appdata/logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'appdata/logs/combined.log' }),
new winston.transports.Console({level: 'info'})
]
});
// var GithubContent = require('github-content'); // var GithubContent = require('github-content');
// Set some defaults // Set some defaults
@@ -67,7 +90,7 @@ var updaterStatus = null;
// check if debug mode // check if debug mode
let debugMode = process.env.YTDL_MODE === 'debug'; let debugMode = process.env.YTDL_MODE === 'debug';
if (debugMode) console.log('YTDL-Material in debug mode!'); if (debugMode) logger.info('YTDL-Material in debug mode!');
// check if just updated // check if just updated
const just_restarted = fs.existsSync('restart.json'); const just_restarted = fs.existsSync('restart.json');
@@ -139,13 +162,13 @@ async function startServer() {
if (usingEncryption) if (usingEncryption)
{ {
https.createServer(options, app).listen(backendPort, function() { https.createServer(options, app).listen(backendPort, function() {
console.log(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on port ${backendPort} - using SSL`); logger.info(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on port ${backendPort} - using SSL`);
}); });
} }
else else
{ {
app.listen(backendPort,function(){ app.listen(backendPort,function(){
console.log(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on PORT ${backendPort}`); logger.info(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on PORT ${backendPort}`);
}); });
} }
@@ -159,7 +182,7 @@ async function restartServer() {
}).unref() }).unref()
process.exit() process.exit()
} }
console.log('Update complete! Restarting server...'); logger.info('Update complete! Restarting server...');
// the following line restarts the server through nodemon // the following line restarts the server through nodemon
fs.writeFileSync('restart.json', 'internal use only'); fs.writeFileSync('restart.json', 'internal use only');
@@ -170,7 +193,7 @@ async function updateServer(tag) {
if (!tag) { if (!tag) {
const new_version_available = await isNewVersionAvailable(); const new_version_available = await isNewVersionAvailable();
if (!new_version_available) { if (!new_version_available) {
console.log('ERROR: Failed to update - no update is available.'); logger.error('ERROR: Failed to update - no update is available.');
return false; return false;
} }
} }
@@ -218,7 +241,7 @@ async function updateServer(tag) {
async function downloadReleaseFiles(tag) { async function downloadReleaseFiles(tag) {
tag = tag ? tag : await getLatestVersion(); tag = tag ? tag : await getLatestVersion();
return new Promise(async resolve => { return new Promise(async resolve => {
console.log('Downloading new files...') logger.info('Downloading new files...')
// downloads the latest release zip file // downloads the latest release zip file
await downloadReleaseZip(tag); await downloadReleaseZip(tag);
@@ -229,7 +252,7 @@ async function downloadReleaseFiles(tag) {
let replace_ignore_list = ['youtubedl-material/appdata/default.json', let replace_ignore_list = ['youtubedl-material/appdata/default.json',
'youtubedl-material/appdata/db.json'] 'youtubedl-material/appdata/db.json']
console.log(`Installing update ${tag}...`) logger.info(`Installing update ${tag}...`)
// downloads new package.json and adds new public dir files from the downloaded zip // downloads new package.json and adds new public dir files from the downloaded zip
fs.createReadStream(path.join(__dirname, `youtubedl-material-latest-release-${tag}.zip`)).pipe(unzipper.Parse()) fs.createReadStream(path.join(__dirname, `youtubedl-material-latest-release-${tag}.zip`)).pipe(unzipper.Parse())
@@ -250,7 +273,7 @@ async function downloadReleaseFiles(tag) {
} else if (!is_dir && !replace_ignore_list.includes(fileName)) { } else if (!is_dir && !replace_ignore_list.includes(fileName)) {
// get package.json // get package.json
var actualFileName = fileName.replace('youtubedl-material/', ''); var actualFileName = fileName.replace('youtubedl-material/', '');
if (debugMode) console.log('Downloading file ' + actualFileName); if (debugMode) logger.verbose('Downloading file ' + actualFileName);
entry.pipe(fs.createWriteStream(path.join(__dirname, actualFileName))); entry.pipe(fs.createWriteStream(path.join(__dirname, actualFileName)));
} else { } else {
entry.autodrain(); entry.autodrain();
@@ -291,7 +314,6 @@ async function fetchFile(url, path, file_label) {
} }
async function downloadReleaseZip(tag) { async function downloadReleaseZip(tag) {
console.log('downloading');
return new Promise(async resolve => { return new Promise(async resolve => {
// get name of zip file, which depends on the version // get name of zip file, which depends on the version
const latest_release_link = `https://github.com/Tzahi12345/YoutubeDL-Material/releases/download/${tag}/`; const latest_release_link = `https://github.com/Tzahi12345/YoutubeDL-Material/releases/download/${tag}/`;
@@ -319,7 +341,7 @@ async function installDependencies() {
async function backupServerLite() { async function backupServerLite() {
return new Promise(async resolve => { return new Promise(async resolve => {
let output_path = `backup-${Date.now()}.zip`; let output_path = `backup-${Date.now()}.zip`;
console.log(`Backing up your non-video/audio files to ${output_path}. This may take up to a few seconds/minutes.`); logger.info(`Backing up your non-video/audio files to ${output_path}. This may take up to a few seconds/minutes.`);
let output = fs.createWriteStream(path.join(__dirname, output_path)); let output = fs.createWriteStream(path.join(__dirname, output_path));
var archive = archiver('zip', { var archive = archiver('zip', {
gzip: true, gzip: true,
@@ -327,7 +349,7 @@ async function backupServerLite() {
}); });
archive.on('error', function(err) { archive.on('error', function(err) {
console.log(err); logger.error(err);
resolve(false); resolve(false);
}); });
@@ -373,9 +395,9 @@ async function getLatestVersion() {
.then(async (json) => { .then(async (json) => {
if (json['message']) { if (json['message']) {
// means there's an error in getting latest version // means there's an error in getting latest version
console.log(`ERROR: Received the following message from GitHub's API:`); logger.error(`ERROR: Received the following message from GitHub's API:`);
console.log(json['message']); logger.error(json['message']);
if (json['documentation_url']) console.log(`Associated URL: ${json['documentation_url']}`) if (json['documentation_url']) logger.error(`Associated URL: ${json['documentation_url']}`)
} }
resolve(json['tag_name']); resolve(json['tag_name']);
return; return;
@@ -400,10 +422,10 @@ async function setConfigFromEnv() {
let config_items = getEnvConfigItems(); let config_items = getEnvConfigItems();
let success = config_api.setConfigItems(config_items); let success = config_api.setConfigItems(config_items);
if (success) { if (success) {
console.log('Config items set using ENV variables.'); logger.info('Config items set using ENV variables.');
setTimeout(() => resolve(true), 100); setTimeout(() => resolve(true), 100);
} else { } else {
console.log('ERROR: Failed to set config items using ENV variables.'); logger.error('ERROR: Failed to set config items using ENV variables.');
resolve(false); resolve(false);
} }
}); });
@@ -423,7 +445,7 @@ async function loadConfig() {
subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval'); subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`) logger.info(`Using non-default downloading agent \'${customDownloadingAgent}\'`)
} else { } else {
customDownloadingAgent = null; customDownloadingAgent = null;
} }
@@ -485,7 +507,7 @@ function watchSubscriptions() {
let current_delay = 0; let current_delay = 0;
for (let i = 0; i < subscriptions.length; i++) { for (let i = 0; i < subscriptions.length; i++) {
let sub = subscriptions[i]; let sub = subscriptions[i];
if (debugMode) console.log('watching ' + sub.name + ' with delay interval of ' + delay_interval); logger.debug('watching ' + sub.name + ' with delay interval of ' + delay_interval);
setTimeout(() => { setTimeout(() => {
subscriptions_api.getVideosForSub(sub); subscriptions_api.getVideosForSub(sub);
}, current_delay); }, current_delay);
@@ -655,7 +677,7 @@ async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvid
}); });
archive.on('error', function(err) { archive.on('error', function(err) {
console.log(err); logger.error(err);
throw err; throw err;
}); });
@@ -717,7 +739,7 @@ async function deleteAudioFile(name, blacklistMode = false) {
const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) writeToBlacklist('audio', line); if (blacklistMode && line) writeToBlacklist('audio', line);
} else { } else {
console.log('Could not find archive file for audio files. Creating...'); logger.info('Could not find archive file for audio files. Creating...');
fs.closeSync(fs.openSync(archive_path, 'w')); fs.closeSync(fs.openSync(archive_path, 'w'));
} }
} }
@@ -775,7 +797,7 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) writeToBlacklist('video', line); if (blacklistMode && line) writeToBlacklist('video', line);
} else { } else {
console.log('Could not find archive file for videos. Creating...'); logger.info('Could not find archive file for videos. Creating...');
fs.closeSync(fs.openSync(archive_path, 'w')); fs.closeSync(fs.openSync(archive_path, 'w'));
} }
} }
@@ -831,7 +853,7 @@ function getAudioInfos(fileNames) {
try { try {
result.push(JSON.parse(data)); result.push(JSON.parse(data));
} catch(e) { } catch(e) {
console.log(`ERROR: Could not find info for file ${fileName}.mp3`); logger.error(`Could not find info for file ${fileName}.mp3`);
} }
} }
} }
@@ -848,7 +870,7 @@ function getVideoInfos(fileNames) {
try { try {
result.push(JSON.parse(data)); result.push(JSON.parse(data));
} catch(e) { } catch(e) {
console.log(`ERROR: Could not find info for file ${fileName}.mp4`); logger.error(`Could not find info for file ${fileName}.mp4`);
} }
} }
} }
@@ -864,10 +886,10 @@ async function getUrlInfos(urls) {
if (debugMode) { if (debugMode) {
let new_date = Date.now(); let new_date = Date.now();
let difference = (new_date - startDate)/1000; let difference = (new_date - startDate)/1000;
console.log(`URL info retrieval delay: ${difference} seconds.`); logger.info(`URL info retrieval delay: ${difference} seconds.`);
} }
if (err) { if (err) {
console.log('Error during parsing:' + err); logger.error('Error during parsing:' + err);
resolve(null); resolve(null);
} }
let try_putput = null; let try_putput = null;
@@ -876,8 +898,8 @@ async function getUrlInfos(urls) {
result = try_putput; result = try_putput;
} catch(e) { } catch(e) {
// probably multiple urls // probably multiple urls
console.log('failed to parse for urls starting with ' + urls[0]); logger.error('failed to parse for urls starting with ' + urls[0]);
// console.log(output); // logger.info(output);
} }
resolve(result); resolve(result);
}); });
@@ -903,7 +925,7 @@ async function autoUpdateYoutubeDL() {
let current_app_details_path = 'node_modules/youtube-dl/bin/details'; let current_app_details_path = 'node_modules/youtube-dl/bin/details';
let current_app_details_exists = fs.existsSync(current_app_details_path); let current_app_details_exists = fs.existsSync(current_app_details_path);
if (!current_app_details_exists) { if (!current_app_details_exists) {
console.log(`ERROR: Failed to get youtube-dl binary details at location '${current_app_details_path}'. Cancelling update check.`); logger.error(`Failed to get youtube-dl binary details at location '${current_app_details_path}'. Cancelling update check.`);
resolve(false); resolve(false);
return; return;
} }
@@ -911,14 +933,14 @@ async function autoUpdateYoutubeDL() {
let current_version = current_app_details['version']; let current_version = current_app_details['version'];
let stored_binary_path = current_app_details['path']; let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') { if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// console.log(`INFO: Failed to get youtube-dl binary path at location: ${current_app_details_path}, attempting to guess actual path...`); // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${current_app_details_path}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/'; const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (process.platform === 'win32' ? '.exe' : ''); const guessed_file_path = guessed_base_path + 'youtube-dl' + (process.platform === 'win32' ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) { if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path; stored_binary_path = guessed_file_path;
// console.log('INFO: Guess successful! Update process continuing...') // logger.info('INFO: Guess successful! Update process continuing...')
} else { } else {
console.log(`ERROR: Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(false); resolve(false);
return; return;
} }
@@ -938,11 +960,11 @@ async function autoUpdateYoutubeDL() {
if (current_version !== latest_update_version) { if (current_version !== latest_update_version) {
let binary_path = 'node_modules/youtube-dl/bin'; let binary_path = 'node_modules/youtube-dl/bin';
// versions different, download new update // versions different, download new update
console.log('INFO: Found new update for youtube-dl. Updating binary...'); logger.info('Found new update for youtube-dl. Updating binary...');
try { try {
await checkExistsWithTimeout(stored_binary_path, 10000); await checkExistsWithTimeout(stored_binary_path, 10000);
} catch(e) { } catch(e) {
console.log(`ERROR: Failed to update youtube-dl - ${e}`); logger.error(`Failed to update youtube-dl - ${e}`);
} }
downloader(binary_path, function error(err, done) { downloader(binary_path, function error(err, done) {
'use strict' 'use strict'
@@ -950,7 +972,7 @@ async function autoUpdateYoutubeDL() {
resolve(false); resolve(false);
throw err; throw err;
} }
console.log(`INFO: Binary successfully updated: ${current_version} -> ${latest_update_version}`); logger.info(`Binary successfully updated: ${current_version} -> ${latest_update_version}`);
resolve(true); resolve(true);
}); });
} }
@@ -1026,7 +1048,7 @@ app.post('/api/setConfig', function(req, res) {
success: success success: success
}); });
} else { } else {
console.log('ERROR: Tried to save invalid config file!') logger.error('Tried to save invalid config file!')
res.sendStatus(400); res.sendStatus(400);
} }
@@ -1115,14 +1137,12 @@ app.post('/api/tomp3', async function(req, res) {
} }
youtubedl.exec(url, downloadConfig, {}, function(err, output) { youtubedl.exec(url, downloadConfig, {}, function(err, output) {
if (debugMode) { let new_date = Date.now();
let new_date = Date.now(); let difference = (new_date - date)/1000;
let difference = (new_date - date)/1000; logger.debug(`Audio download delay: ${difference} seconds.`);
console.log(`Audio download delay: ${difference} seconds.`);
}
if (err) { if (err) {
audiopath = "-1"; audiopath = "-1";
console.log(err.stderr); logger.error(err.stderr);
res.sendStatus(500); res.sendStatus(500);
throw err; throw err;
} else if (output) { } else if (output) {
@@ -1151,9 +1171,9 @@ app.post('/api/tomp3', async function(req, res) {
} }
// NodeID3.create(tags, function(frame) { }) // NodeID3.create(tags, function(frame) { })
let success = NodeID3.write(tags, full_file_path); let success = NodeID3.write(tags, full_file_path);
if (!success) console.log('ERROR: Failed to apply ID3 tag to audio file ' + full_file_path); if (!success) logger.error('Failed to apply ID3 tag to audio file ' + full_file_path);
} else { } else {
console.log('Output mp3 does not exist'); logger.info('Output mp3 does not exist');
} }
var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length-5); var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length-5);
@@ -1252,14 +1272,12 @@ app.post('/api/tomp4', async function(req, res) {
} }
youtubedl.exec(url, downloadConfig, {}, function(err, output) { youtubedl.exec(url, downloadConfig, {}, function(err, output) {
if (debugMode) { let new_date = Date.now();
let new_date = Date.now(); let difference = (new_date - date)/1000;
let difference = (new_date - date)/1000; logger.debug(`Video download delay: ${difference} seconds.`);
console.log(`Video download delay: ${difference} seconds.`);
}
if (err) { if (err) {
videopath = "-1"; videopath = "-1";
console.log(err.stderr); logger.error(err.stderr);
res.sendStatus(500); res.sendStatus(500);
throw err; throw err;
} else if (output) { } else if (output) {
@@ -1285,7 +1303,7 @@ app.post('/api/tomp4', async function(req, res) {
if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
try { try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
console.log('Renamed ' + file_name + '.webm to ' + file_name); logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) { } catch(e) {
} }
} }
@@ -1333,7 +1351,7 @@ app.post('/api/fileStatusMp3', function(req, res) {
percent = downloaded/size; percent = downloaded/size;
exists = ["failed", getFileSizeMp3(name), percent]; exists = ["failed", getFileSizeMp3(name), percent];
} }
//console.log(exists + " " + name); //logger.info(exists + " " + name);
res.send(exists); res.send(exists);
res.end("yes"); res.end("yes");
}); });
@@ -1353,7 +1371,7 @@ app.post('/api/fileStatusMp4', function(req, res) {
percent = downloaded/size; percent = downloaded/size;
exists = ["failed", getFileSizeMp4(name), percent]; exists = ["failed", getFileSizeMp4(name), percent];
} }
//console.log(exists + " " + name); //logger.info(exists + " " + name);
res.send(exists); res.send(exists);
res.end("yes"); res.end("yes");
}); });
@@ -1516,7 +1534,7 @@ app.post('/api/getSubscription', async (req, res) => {
files = recFindByExt(appended_base_path, 'mp4'); files = recFindByExt(appended_base_path, 'mp4');
} catch(e) { } catch(e) {
files = null; files = null;
console.log('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);
res.sendStatus(500); res.sendStatus(500);
return; return;
} }
@@ -1605,11 +1623,11 @@ app.post('/api/updatePlaylist', async (req, res) => {
.find({id: playlistID}) .find({id: playlistID})
.assign({fileNames: fileNames}) .assign({fileNames: fileNames})
.write(); .write();
/*console.log('success!'); /*logger.info('success!');
let new_val = db.get(`playlists.${type}`) let new_val = db.get(`playlists.${type}`)
.find({id: playlistID}) .find({id: playlistID})
.value(); .value();
console.log(new_val);*/ logger.info(new_val);*/
success = true; success = true;
} catch(e) { } catch(e) {
console.error(`Failed to find playlist with ID ${playlistID}`); console.error(`Failed to find playlist with ID ${playlistID}`);
@@ -1711,7 +1729,7 @@ app.post('/api/downloadFile', async (req, res) => {
try { try {
fs.unlinkSync(file); fs.unlinkSync(file);
} catch(e) { } catch(e) {
console.log("ERROR: Failed to remove file", file); logger.error("Failed to remove file", file);
} }
} }
}); });
@@ -1836,7 +1854,7 @@ app.get('/api/video/:id', function(req , res){
file.on('close', function() { file.on('close', function() {
let index = descriptors[id].indexOf(file); let index = descriptors[id].indexOf(file);
descriptors[id].splice(index, 1); descriptors[id].splice(index, 1);
if (debugMode) console.log('Successfully closed stream and removed file reference.'); logger.debug('Successfully closed stream and removed file reference.');
}); });
head = { head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`, 'Content-Range': `bytes ${start}-${end}/${fileSize}`,
@@ -1877,7 +1895,7 @@ app.get('/api/audio/:id', function(req , res){
file.on('close', function() { file.on('close', function() {
let index = descriptors[id].indexOf(file); let index = descriptors[id].indexOf(file);
descriptors[id].splice(index, 1); descriptors[id].splice(index, 1);
if (debugMode) console.log('Successfully closed stream and removed file reference.'); logger.debug('Successfully closed stream and removed file reference.');
}); });
head = { head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`, 'Content-Range': `bytes ${start}-${end}/${fileSize}`,

1
backend/package.json
View File

@@ -45,6 +45,7 @@
"shortid": "^2.2.15", "shortid": "^2.2.15",
"unzipper": "^0.10.10", "unzipper": "^0.10.10",
"uuidv4": "^6.0.6", "uuidv4": "^6.0.6",
"winston": "^3.2.1",
"youtube-dl": "^3.0.2" "youtube-dl": "^3.0.2"
} }
} }