TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-17 09:41:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #216 from NotWoods/await

Browse Source 
Use async versions of filesystem methods
This commit is contained in:
Tzahi12345
2020-09-29 17:41:19 -04:00
committed by GitHub
parent b8280e8646 3e4e7edd90
commit fb72dee26f
5 changed files with 687 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

398
backend/app.js
View File

@@ -1,6 +1,6 @@
var async = require('async');
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
var fs = require('fs-extra'); var fs = require('fs-extra');
var { promisify } = require('util');
var auth_api = require('./authentication/auth'); var auth_api = require('./authentication/auth');
var winston = require('winston'); var winston = require('winston');
var path = require('path'); var path = require('path');
@@ -18,7 +18,6 @@ var utils = require('./utils')
var mergeFiles = require('merge-files'); var mergeFiles = require('merge-files');
const low = require('lowdb') const low = require('lowdb')
var ProgressBar = require('progress'); var ProgressBar = require('progress');
var md5 = require('md5');
const NodeID3 = require('node-id3') const NodeID3 = require('node-id3')
const downloader = require('youtube-dl/lib/downloader') const downloader = require('youtube-dl/lib/downloader')
const fetch = require('node-fetch'); const fetch = require('node-fetch');
@@ -195,35 +194,41 @@ app.use(auth_api.passport.initialize());
// actual functions // actual functions
/**
* setTimeout, but its a promise.
* @param {number} ms
*/
async function wait(ms) {
await new Promise(resolve => {
setTimeout(resolve, ms);
});
}
async function checkMigrations() { async function checkMigrations() {
return new Promise(async resolve => {
// 3.5->3.6 migration // 3.5->3.6 migration
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
if (!files_to_db_migration_complete) { if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+') logger.info('Beginning migration: 3.5->3.6+')
runFilesToDBMigration().then(success => { const success = await runFilesToDBMigration()
if (success) { logger.info('3.5->3.6+ migration complete!'); } if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); } else { logger.error('Migration failed: 3.5->3.6+'); }
});
} }
resolve(true); return true;
});
} }
async function runFilesToDBMigration() { async function runFilesToDBMigration() {
return new Promise(async resolve => {
try { try {
let mp3s = getMp3s(); let mp3s = await getMp3s();
let mp4s = getMp4s(); let mp4s = await getMp4s();
for (let i = 0; i < mp3s.length; i++) { for (let i = 0; i < mp3s.length; i++) {
let file_obj = mp3s[i]; let file_obj = mp3s[i];
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) { if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`); logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); await db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
} }
} }
@@ -232,18 +237,17 @@ async function runFilesToDBMigration() {
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) { if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`); logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp4', 'video'); await db_api.registerFileDB(file_obj.id + '.mp4', 'video');
} }
} }
// sets migration to complete // sets migration to complete
db.set('files_to_db_migration_complete', true).write(); db.set('files_to_db_migration_complete', true).write();
resolve(true); return true;
} catch(err) { } catch(err) {
logger.error(err); logger.error(err);
resolve(false); return false;
} }
});
} }
async function startServer() { async function startServer() {
@@ -418,20 +422,20 @@ async function downloadReleaseZip(tag) {
} }
async function installDependencies() { async function installDependencies() {
return new Promise(resolve => {
var child_process = require('child_process'); var child_process = require('child_process');
child_process.execSync('npm install',{stdio:[0,1,2]}); var exec = promisify(child_process.exec);
resolve(true);
});
await exec('npm install',{stdio:[0,1,2]});
return true;
} }
async function backupServerLite() { async function backupServerLite() {
return new Promise(async resolve => { await fs.ensureDir(path.join(__dirname, 'appdata', 'backups'));
fs.ensureDirSync(path.join(__dirname, 'appdata', 'backups'));
let output_path = path.join('appdata', 'backups', `backup-${Date.now()}.zip`); let output_path = path.join('appdata', 'backups', `backup-${Date.now()}.zip`);
logger.info(`Backing up your non-video/audio files to ${output_path}. This may take up to a few seconds/minutes.`); logger.info(`Backing up your non-video/audio files to ${output_path}. This may take up to a few seconds/minutes.`);
let output = fs.createWriteStream(path.join(__dirname, output_path)); let output = fs.createWriteStream(path.join(__dirname, output_path));
await new Promise(resolve => {
var archive = archiver('zip', { var archive = archiver('zip', {
gzip: true, gzip: true,
zlib: { level: 9 } // Sets the compression level. zlib: { level: 9 } // Sets the compression level.
@@ -455,58 +459,53 @@ async function backupServerLite() {
ignore: files_to_ignore ignore: files_to_ignore
}); });
await archive.finalize(); resolve(archive.finalize());
});
// wait a tiny bit for the zip to reload in fs // wait a tiny bit for the zip to reload in fs
setTimeout(function() { await wait(100);
resolve(true); return true;
}, 100);
});
} }
async function isNewVersionAvailable() { async function isNewVersionAvailable() {
return new Promise(async resolve => {
// gets tag of the latest version of youtubedl-material, compare to current version // gets tag of the latest version of youtubedl-material, compare to current version
const latest_tag = await getLatestVersion(); const latest_tag = await getLatestVersion();
const current_tag = CONSTS['CURRENT_VERSION']; const current_tag = CONSTS['CURRENT_VERSION'];
if (latest_tag > current_tag) { if (latest_tag > current_tag) {
resolve(true); return true;
} else { } else {
resolve(false); return false;
} }
});
} }
async function getLatestVersion() { async function getLatestVersion() {
return new Promise(resolve => { const res = await fetch('https://api.github.com/repos/tzahi12345/youtubedl-material/releases/latest', {method: 'Get'});
fetch('https://api.github.com/repos/tzahi12345/youtubedl-material/releases/latest', {method: 'Get'}) const json = await res.json();
.then(async res => res.json())
.then(async (json) => {
if (json['message']) { if (json['message']) {
// means there's an error in getting latest version // means there's an error in getting latest version
logger.error(`ERROR: Received the following message from GitHub's API:`); logger.error(`ERROR: Received the following message from GitHub's API:`);
logger.error(json['message']); logger.error(json['message']);
if (json['documentation_url']) logger.error(`Associated URL: ${json['documentation_url']}`) if (json['documentation_url']) logger.error(`Associated URL: ${json['documentation_url']}`)
} }
resolve(json['tag_name']); return json['tag_name'];
return;
});
});
} }
async function killAllDownloads() { async function killAllDownloads() {
return new Promise(resolve => { const lookupAsync = promisify(ps.lookup);
ps.lookup({
command: 'youtube-dl', try {
}, function(err, resultList ) { await lookupAsync({
if (err) { command: 'youtube-dl'
});
} catch (err) {
// failed to get list of processes // failed to get list of processes
logger.error('Failed to get a list of running youtube-dl processes.'); logger.error('Failed to get a list of running youtube-dl processes.');
logger.error(err); logger.error(err);
resolve({ return {
details: err, details: err,
success: false success: false
}); };
} }
// processes that contain the string 'youtube-dl' in the name will be looped // processes that contain the string 'youtube-dl' in the name will be looped
@@ -524,18 +523,16 @@ async function killAllDownloads() {
}); });
} }
}); });
resolve({
return {
success: true success: true
}); };
});
});
} }
async function setPortItemFromENV() { async function setPortItemFromENV() {
return new Promise(resolve => {
config_api.setConfigItem('ytdl_port', backendPort.toString()); config_api.setConfigItem('ytdl_port', backendPort.toString());
setTimeout(() => resolve(true), 100); await wait(100);
}); return true;
} }
async function setAndLoadConfig() { async function setAndLoadConfig() {
@@ -544,27 +541,23 @@ async function setAndLoadConfig() {
} }
async function setConfigFromEnv() { async function setConfigFromEnv() {
return new Promise(resolve => {
let config_items = getEnvConfigItems(); let config_items = getEnvConfigItems();
let success = config_api.setConfigItems(config_items); let success = config_api.setConfigItems(config_items);
if (success) { if (success) {
logger.info('Config items set using ENV variables.'); logger.info('Config items set using ENV variables.');
setTimeout(() => resolve(true), 100); await wait(100);
return true;
} else { } else {
logger.error('ERROR: Failed to set config items using ENV variables.'); logger.error('ERROR: Failed to set config items using ENV variables.');
resolve(false); return false;
} }
});
} }
async function loadConfig() { async function loadConfig() {
return new Promise(async resolve => {
loadConfigValues(); loadConfigValues();
// creates archive path if missing // creates archive path if missing
if (!fs.existsSync(archivePath)){ await fs.ensureDir(archivePath);
fs.mkdirSync(archivePath);
}
// get subscriptions // get subscriptions
if (allowSubscriptions) { if (allowSubscriptions) {
@@ -586,9 +579,7 @@ async function loadConfig() {
// start the server here // start the server here
startServer(); startServer();
resolve(true); return true;
});
} }
function loadConfigValues() { function loadConfigValues() {
@@ -705,17 +696,17 @@ function generateEnvVarConfigItem(key) {
return {key: key, value: process['env'][key]}; return {key: key, value: process['env'][key]};
} }
function getMp3s() { async function getMp3s() {
let mp3s = []; let mp3s = [];
var files = utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
for (let i = 0; i < files.length; i++) { for (let i = 0; i < files.length; i++) {
let file = files[i]; let file = files[i];
var file_path = file.substring(audioFolderPath.length, file.length); var file_path = file.substring(audioFolderPath.length, file.length);
var stats = fs.statSync(file); var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp3(id, audioFolderPath); var jsonobj = await utils.getJSONMp3(id, audioFolderPath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@@ -734,9 +725,9 @@ function getMp3s() {
return mp3s; return mp3s;
} }
function getMp4s(relative_path = true) { async function getMp4s(relative_path = true) {
let mp4s = []; let mp4s = [];
var files = utils.recFindByExt(videoFolderPath, 'mp4'); var files = await utils.recFindByExt(videoFolderPath, 'mp4');
for (let i = 0; i < files.length; i++) { for (let i = 0; i < files.length; i++) {
let file = files[i]; let file = files[i];
var file_path = file.substring(videoFolderPath.length, file.length); var file_path = file.substring(videoFolderPath.length, file.length);
@@ -744,7 +735,7 @@ function getMp4s(relative_path = true) {
var stats = fs.statSync(file); var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp4(id, videoFolderPath); var jsonobj = await utils.getJSONMp4(id, videoFolderPath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@@ -850,11 +841,10 @@ function getVideoFormatID(name)
} }
async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) {
return new Promise(async resolve => {
let zipFolderPath = null; let zipFolderPath = null;
if (!fullPathProvided) { if (!fullPathProvided) {
zipFolderPath = path.join(type === 'audio' ? audioFolderPath : videoFolderPath); zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath);
if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath);
} else { } else {
zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path'));
@@ -887,17 +877,11 @@ async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvid
await archive.finalize(); await archive.finalize();
// wait a tiny bit for the zip to reload in fs // wait a tiny bit for the zip to reload in fs
setTimeout(function() { await wait(100);
resolve(path.join(zipFolderPath,outputName + '.zip')); return path.join(zipFolderPath,outputName + '.zip');
}, 100);
});
} }
async function deleteAudioFile(name, customPath = null, blacklistMode = false) { async function deleteAudioFile(name, customPath = null, blacklistMode = false) {
return new Promise(resolve => {
let filePath = customPath ? customPath : audioFolderPath; let filePath = customPath ? customPath : audioFolderPath;
var jsonPath = path.join(filePath,name+'.mp3.info.json'); var jsonPath = path.join(filePath,name+'.mp3.info.json');
@@ -910,24 +894,24 @@ async function deleteAudioFile(name, customPath = null, blacklistMode = false) {
altJSONPath = path.join(__dirname, altJSONPath); altJSONPath = path.join(__dirname, altJSONPath);
audioFilePath = path.join(__dirname, audioFilePath); audioFilePath = path.join(__dirname, audioFilePath);
let jsonExists = fs.existsSync(jsonPath); let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = fs.existsSync(thumbnailPath); let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) { if (!jsonExists) {
if (fs.existsSync(altJSONPath)) { if (await fs.pathExists(altJSONPath)) {
jsonExists = true; jsonExists = true;
jsonPath = altJSONPath; jsonPath = altJSONPath;
} }
} }
if (!thumbnailExists) { if (!thumbnailExists) {
if (fs.existsSync(altThumbnailPath)) { if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true; thumbnailExists = true;
thumbnailPath = altThumbnailPath; thumbnailPath = altThumbnailPath;
} }
} }
let audioFileExists = fs.existsSync(audioFilePath); let audioFileExists = await fs.pathExists(audioFilePath);
if (config_api.descriptors[name]) { if (config_api.descriptors[name]) {
try { try {
@@ -945,40 +929,36 @@ async function deleteAudioFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON // get ID from JSON
var jsonobj = utils.getJSONMp3(name, filePath); var jsonobj = await utils.getJSONMp3(name, filePath);
let id = null; let id = null;
if (jsonobj) id = jsonobj.id; if (jsonobj) id = jsonobj.id;
// use subscriptions API to remove video from the archive file, and write it to the blacklist // use subscriptions API to remove video from the archive file, and write it to the blacklist
if (fs.existsSync(archive_path)) { if (await fs.pathExists(archive_path)) {
const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) writeToBlacklist('audio', line); if (blacklistMode && line) await writeToBlacklist('audio', line);
} else { } else {
logger.info('Could not find archive file for audio files. Creating...'); logger.info('Could not find archive file for audio files. Creating...');
fs.closeSync(fs.openSync(archive_path, 'w')); await fs.close(await fs.open(archive_path, 'w'));
} }
} }
if (jsonExists) fs.unlinkSync(jsonPath); if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) fs.unlinkSync(thumbnailPath); if (thumbnailExists) await fs.unlink(thumbnailPath);
if (audioFileExists) { if (audioFileExists) {
fs.unlink(audioFilePath, function(err) { await fs.unlink(audioFilePath);
if (fs.existsSync(jsonPath) || fs.existsSync(audioFilePath)) { if (await fs.pathExists(jsonPath) || await fs.pathExists(audioFilePath)) {
resolve(false); return false;
} else { } else {
resolve(true); return true;
} }
});
} else { } else {
// TODO: tell user that the file didn't exist // TODO: tell user that the file didn't exist
resolve(true); return true;
} }
});
} }
async function deleteVideoFile(name, customPath = null, blacklistMode = false) { async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
return new Promise(resolve => {
let filePath = customPath ? customPath : videoFolderPath; let filePath = customPath ? customPath : videoFolderPath;
var jsonPath = path.join(filePath,name+'.info.json'); var jsonPath = path.join(filePath,name+'.info.json');
@@ -990,19 +970,19 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
jsonPath = path.join(__dirname, jsonPath); jsonPath = path.join(__dirname, jsonPath);
videoFilePath = path.join(__dirname, videoFilePath); videoFilePath = path.join(__dirname, videoFilePath);
let jsonExists = fs.existsSync(jsonPath); let jsonExists = await fs.pathExists(jsonPath);
let videoFileExists = fs.existsSync(videoFilePath); let videoFileExists = await fs.pathExists(videoFilePath);
let thumbnailExists = fs.existsSync(thumbnailPath); let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) { if (!jsonExists) {
if (fs.existsSync(altJSONPath)) { if (await fs.pathExists(altJSONPath)) {
jsonExists = true; jsonExists = true;
jsonPath = altJSONPath; jsonPath = altJSONPath;
} }
} }
if (!thumbnailExists) { if (!thumbnailExists) {
if (fs.existsSync(altThumbnailPath)) { if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true; thumbnailExists = true;
thumbnailPath = altThumbnailPath; thumbnailPath = altThumbnailPath;
} }
@@ -1024,86 +1004,67 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON // get ID from JSON
var jsonobj = utils.getJSONMp4(name, filePath); var jsonobj = await utils.getJSONMp4(name, filePath);
let id = null; let id = null;
if (jsonobj) id = jsonobj.id; if (jsonobj) id = jsonobj.id;
// use subscriptions API to remove video from the archive file, and write it to the blacklist // use subscriptions API to remove video from the archive file, and write it to the blacklist
if (fs.existsSync(archive_path)) { if (await fs.pathExists(archive_path)) {
const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) writeToBlacklist('video', line); if (blacklistMode && line) await writeToBlacklist('video', line);
} else { } else {
logger.info('Could not find archive file for videos. Creating...'); logger.info('Could not find archive file for videos. Creating...');
fs.closeSync(fs.openSync(archive_path, 'w')); fs.closeSync(fs.openSync(archive_path, 'w'));
} }
} }
if (jsonExists) fs.unlinkSync(jsonPath); if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) fs.unlinkSync(thumbnailPath); if (thumbnailExists) await fs.unlink(thumbnailPath);
if (videoFileExists) { if (videoFileExists) {
fs.unlink(videoFilePath, function(err) { await fs.unlink(videoFilePath);
if (fs.existsSync(jsonPath) || fs.existsSync(videoFilePath)) { if (await fs.pathExists(jsonPath) || await fs.pathExists(videoFilePath)) {
resolve(false); return false;
} else { } else {
resolve(true); return true;
} }
});
} else { } else {
// TODO: tell user that the file didn't exist // TODO: tell user that the file didn't exist
resolve(true); return true;
} }
});
} }
// replaces .webm with appropriate extension /**
function getTrueFileName(unfixed_path, type) { * @param {'audio' | 'video'} type
let fixed_path = unfixed_path; * @param {string[]} fileNames
*/
const new_ext = (type === 'audio' ? 'mp3' : 'mp4'); async function getAudioOrVideoInfos(type, fileNames) {
let unfixed_parts = unfixed_path.split('.'); let result = await Promise.all(fileNames.map(async fileName => {
const old_ext = unfixed_parts[unfixed_parts.length-1]; let fileLocation = videoFolderPath+fileName;
if (type === 'audio') {
fileLocation += '.mp3.info.json';
if (old_ext !== new_ext) { } else if (type === 'video') {
unfixed_parts[unfixed_parts.length-1] = new_ext; fileLocation += '.info.json';
fixed_path = unfixed_parts.join('.');
} }
return fixed_path;
}
function getAudioInfos(fileNames) { if (await fs.pathExists(fileLocation)) {
let result = []; let data = await fs.readFile(fileLocation);
for (let i = 0; i < fileNames.length; i++) {
let fileName = fileNames[i];
let fileLocation = audioFolderPath+fileName+'.mp3.info.json';
if (fs.existsSync(fileLocation)) {
let data = fs.readFileSync(fileLocation);
try { try {
result.push(JSON.parse(data)); return JSON.parse(data);
} catch(e) { } catch (e) {
logger.error(`Could not find info for file ${fileName}.mp3`); let suffix;
if (type === 'audio') {
suffix += '.mp3';
} else if (type === 'video') {
suffix += '.mp4';
} }
}
}
return result;
}
function getVideoInfos(fileNames) { logger.error(`Could not find info for file ${fileName}${suffix}`);
let result = [];
for (let i = 0; i < fileNames.length; i++) {
let fileName = fileNames[i];
let fileLocation = videoFolderPath+fileName+'.info.json';
if (fs.existsSync(fileLocation)) {
let data = fs.readFileSync(fileLocation);
try {
result.push(JSON.parse(data));
} catch(e) {
logger.error(`Could not find info for file ${fileName}.mp4`);
} }
} }
} return null;
return result; }));
return result.filter(data => data != null);
} }
// downloads // downloads
@@ -1401,7 +1362,6 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) {
} }
async function generateArgs(url, type, options) { async function generateArgs(url, type, options) {
return new Promise(async resolve => {
var videopath = '%(title)s'; var videopath = '%(title)s';
var globalArgs = config_api.getConfigItem('ytdl_custom_args'); var globalArgs = config_api.getConfigItem('ytdl_custom_args');
let useCookies = config_api.getConfigItem('ytdl_use_cookies'); let useCookies = config_api.getConfigItem('ytdl_use_cookies');
@@ -1463,7 +1423,7 @@ async function generateArgs(url, type, options) {
} }
if (useCookies) { if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) { if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else { } else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -1479,26 +1439,26 @@ async function generateArgs(url, type, options) {
const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath; const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath;
const archive_path = path.join(archive_folder, `archive_${type}.txt`); const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.ensureDirSync(archive_folder); await fs.ensureDir(archive_folder);
// create archive file if it doesn't exist // create archive file if it doesn't exist
if (!fs.existsSync(archive_path)) { if (!(await fs.pathExists(archive_path))) {
fs.closeSync(fs.openSync(archive_path, 'w')); await fs.close(await fs.open(archive_path, 'w'));
} }
let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
// create blacklist file if it doesn't exist // create blacklist file if it doesn't exist
if (!fs.existsSync(blacklist_path)) { if (!(await fs.pathExists(blacklist_path))) {
fs.closeSync(fs.openSync(blacklist_path, 'w')); await fs.close(await fs.open(blacklist_path, 'w'));
} }
let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
fs.ensureFileSync(merged_path); await fs.ensureFile(merged_path);
// merges blacklist and regular archive // merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path]; let inputPathList = [archive_path, blacklist_path];
let status = await mergeFiles(inputPathList, merged_path); let status = await mergeFiles(inputPathList, merged_path);
options.merged_string = fs.readFileSync(merged_path, "utf8"); options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path); downloadConfig.push('--download-archive', merged_path);
} }
@@ -1519,8 +1479,7 @@ async function generateArgs(url, type, options) {
} }
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
resolve(downloadConfig); return downloadConfig;
});
} }
async function getVideoInfoByURL(url, args = [], download = null) { async function getVideoInfoByURL(url, args = [], download = null) {
@@ -1593,11 +1552,11 @@ async function convertFileToMp3(input_file, output_file) {
}); });
} }
function writeToBlacklist(type, line) { async function writeToBlacklist(type, line) {
let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
// adds newline to the beginning of the line // adds newline to the beginning of the line
line = '\n' + line; line = '\n' + line;
fs.appendFileSync(blacklistPath, line); await fs.appendFile(blacklistPath, line);
} }
// download management functions // download management functions
@@ -1741,21 +1700,6 @@ function removeFileExtension(filename) {
return filename_parts.join('.'); return filename_parts.join('.');
} }
// https://stackoverflow.com/a/32197381/8088021
const deleteFolderRecursive = function(folder_to_delete) {
if (fs.existsSync(folder_to_delete)) {
fs.readdirSync(folder_to_delete).forEach((file, index) => {
const curPath = path.join(folder_to_delete, file);
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);
} else { // delete file
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(folder_to_delete);
}
};
app.use(function(req, res, next) { app.use(function(req, res, next) {
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization");
res.header("Access-Control-Allow-Origin", getOrigin()); res.header("Access-Control-Allow-Origin", getOrigin());
@@ -1771,8 +1715,12 @@ app.use(function(req, res, next) {
next(); next();
} else if (req.query.apiKey === admin_token) { } else if (req.query.apiKey === admin_token) {
next(); next();
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) { } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key')) {
if (req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
next(); next();
} else {
res.status(401).send('Invalid API key');
}
} else if (req.path.includes('/api/video/') || req.path.includes('/api/audio/')) { } else if (req.path.includes('/api/video/') || req.path.includes('/api/audio/')) {
next(); next();
} else { } else {
@@ -1900,8 +1848,21 @@ app.post('/api/killAllDownloads', optionalJwt, async function(req, res) {
res.send(result_obj); res.send(result_obj);
}); });
/**
* add thumbnails if present
* @param files - List of files with thumbnailPath property.
*/
async function addThumbnails(files) {
await Promise.all(files.map(async file => {
const thumbnailPath = file['thumbnailPath'];
if (thumbnailPath && (await fs.pathExists(thumbnailPath))) {
file['thumbnailBlob'] = await fs.readFile(thumbnailPath);
}
}));
}
// gets all download mp3s // gets all download mp3s
app.get('/api/getMp3s', optionalJwt, function(req, res) { app.get('/api/getMp3s', optionalJwt, async function(req, res) {
var mp3s = db.get('files.audio').value(); // getMp3s(); var mp3s = db.get('files.audio').value(); // getMp3s();
var playlists = db.get('playlists.audio').value(); var playlists = db.get('playlists.audio').value();
const is_authenticated = req.isAuthenticated(); const is_authenticated = req.isAuthenticated();
@@ -1915,10 +1876,7 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
mp3s = JSON.parse(JSON.stringify(mp3s)); mp3s = JSON.parse(JSON.stringify(mp3s));
// add thumbnails if present // add thumbnails if present
mp3s.forEach(mp3 => { await addThumbnails(mp3s);
if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath']))
mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']);
});
res.send({ res.send({
mp3s: mp3s, mp3s: mp3s,
@@ -1927,7 +1885,7 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
}); });
// gets all download mp4s // gets all download mp4s
app.get('/api/getMp4s', optionalJwt, function(req, res) { app.get('/api/getMp4s', optionalJwt, async function(req, res) {
var mp4s = db.get('files.video').value(); // getMp4s(); var mp4s = db.get('files.video').value(); // getMp4s();
var playlists = db.get('playlists.video').value(); var playlists = db.get('playlists.video').value();
@@ -1942,10 +1900,7 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) {
mp4s = JSON.parse(JSON.stringify(mp4s)); mp4s = JSON.parse(JSON.stringify(mp4s));
// add thumbnails if present // add thumbnails if present
mp4s.forEach(mp4 => { await addThumbnails(mp4s);
if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath']))
mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']);
});
res.send({ res.send({
mp4s: mp4s, mp4s: mp4s,
@@ -1991,7 +1946,7 @@ app.post('/api/getFile', optionalJwt, function (req, res) {
} }
}); });
app.post('/api/getAllFiles', optionalJwt, function (req, res) { app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
// these are returned // these are returned
let files = []; let files = [];
let playlists = []; let playlists = [];
@@ -2034,10 +1989,7 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) {
files = JSON.parse(JSON.stringify(files)); files = JSON.parse(JSON.stringify(files));
// add thumbnails if present // add thumbnails if present
files.forEach(file => { await addThumbnails(files);
if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath']))
file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']);
});
res.send({ res.send({
files: files, files: files,
@@ -2241,7 +2193,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files; let files;
try { try {
files = utils.recFindByExt(appended_base_path, 'mp4'); files = await utils.recFindByExt(appended_base_path, 'mp4');
} catch(e) { } catch(e) {
files = null; files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);
@@ -2456,7 +2408,7 @@ app.post('/api/deleteMp3', optionalJwt, async (req, res) => {
var blacklistMode = req.body.blacklistMode; var blacklistMode = req.body.blacklistMode;
if (req.isAuthenticated()) { if (req.isAuthenticated()) {
let success = auth_api.deleteUserFile(req.user.uid, uid, 'audio', blacklistMode); let success = await auth_api.deleteUserFile(req.user.uid, uid, 'audio', blacklistMode);
res.send(success); res.send(success);
return; return;
} }
@@ -2465,7 +2417,7 @@ app.post('/api/deleteMp3', optionalJwt, async (req, res) => {
var name = audio_obj.id; var name = audio_obj.id;
var fullpath = audioFolderPath + name + ".mp3"; var fullpath = audioFolderPath + name + ".mp3";
var wasDeleted = false; var wasDeleted = false;
if (fs.existsSync(fullpath)) if (await fs.pathExists(fullpath))
{ {
deleteAudioFile(name, null, blacklistMode); deleteAudioFile(name, null, blacklistMode);
db.get('files.audio').remove({uid: uid}).write(); db.get('files.audio').remove({uid: uid}).write();
@@ -2487,7 +2439,7 @@ app.post('/api/deleteMp4', optionalJwt, async (req, res) => {
var blacklistMode = req.body.blacklistMode; var blacklistMode = req.body.blacklistMode;
if (req.isAuthenticated()) { if (req.isAuthenticated()) {
let success = auth_api.deleteUserFile(req.user.uid, uid, 'video', blacklistMode); let success = await auth_api.deleteUserFile(req.user.uid, uid, 'video', blacklistMode);
res.send(success); res.send(success);
return; return;
} }
@@ -2496,7 +2448,7 @@ app.post('/api/deleteMp4', optionalJwt, async (req, res) => {
var name = video_obj.id; var name = video_obj.id;
var fullpath = videoFolderPath + name + ".mp4"; var fullpath = videoFolderPath + name + ".mp4";
var wasDeleted = false; var wasDeleted = false;
if (fs.existsSync(fullpath)) if (await fs.pathExists(fullpath))
{ {
wasDeleted = await deleteVideoFile(name, null, blacklistMode); wasDeleted = await deleteVideoFile(name, null, blacklistMode);
db.get('files.video').remove({uid: uid}).write(); db.get('files.video').remove({uid: uid}).write();
@@ -2582,7 +2534,7 @@ app.post('/api/downloadArchive', async (req, res) => {
let full_archive_path = path.join(archive_dir, 'archive.txt'); let full_archive_path = path.join(archive_dir, 'archive.txt');
if (fs.existsSync(full_archive_path)) { if (await fs.pathExists(full_archive_path)) {
res.sendFile(full_archive_path); res.sendFile(full_archive_path);
} else { } else {
res.sendStatus(404); res.sendStatus(404);
@@ -2594,14 +2546,14 @@ var upload_multer = multer({ dest: __dirname + '/appdata/' });
app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => { app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
const new_path = path.join(__dirname, 'appdata', 'cookies.txt'); const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
if (fs.existsSync(req.file.path)) { if (await fs.pathExists(req.file.path)) {
fs.renameSync(req.file.path, new_path); await fs.rename(req.file.path, new_path);
} else { } else {
res.sendStatus(500); res.sendStatus(500);
return; return;
} }
if (fs.existsSync(new_path)) { if (await fs.pathExists(new_path)) {
res.send({success: true}); res.send({success: true});
} else { } else {
res.sendStatus(500); res.sendStatus(500);
@@ -2824,9 +2776,9 @@ app.post('/api/logs', async function(req, res) {
let logs = null; let logs = null;
let lines = req.body.lines; let lines = req.body.lines;
logs_path = path.join('appdata', 'logs', 'combined.log') logs_path = path.join('appdata', 'logs', 'combined.log')
if (fs.existsSync(logs_path)) { if (await fs.pathExists(logs_path)) {
if (lines) logs = await read_last_lines.read(logs_path, lines); if (lines) logs = await read_last_lines.read(logs_path, lines);
else logs = fs.readFileSync(logs_path, 'utf8'); else logs = await fs.readFile(logs_path, 'utf8');
} }
else else
logger.error(`Failed to find logs file at the expected location: ${logs_path}`) logger.error(`Failed to find logs file at the expected location: ${logs_path}`)
@@ -2842,8 +2794,10 @@ app.post('/api/clearAllLogs', async function(req, res) {
logs_err_path = path.join('appdata', 'logs', 'error.log'); logs_err_path = path.join('appdata', 'logs', 'error.log');
let success = false; let success = false;
try { try {
fs.writeFileSync(logs_path, ''); await Promise.all([
fs.writeFileSync(logs_err_path, ''); fs.writeFile(logs_path, ''),
fs.writeFile(logs_err_path, '')
])
success = true; success = true;
} catch(e) { } catch(e) {
logger.error(e); logger.error(e);
@@ -2860,10 +2814,8 @@ app.post('/api/clearAllLogs', async function(req, res) {
let type = req.body.type; let type = req.body.type;
let result = null; let result = null;
if (!urlMode) { if (!urlMode) {
if (type === 'audio') { if (type === 'audio' || type === 'video') {
result = getAudioInfos(fileNames) result = await getAudioOrVideoInfos(type, fileNames);
} else if (type === 'video') {
result = getVideoInfos(fileNames);
} }
} else { } else {
result = await getUrlInfos(fileNames); result = await getUrlInfos(fileNames);
@@ -2936,7 +2888,7 @@ app.post('/api/deleteUser', optionalJwt, async (req, res) => {
const user_db_obj = users_db.get('users').find({uid: uid}); const user_db_obj = users_db.get('users').find({uid: uid});
if (user_db_obj.value()) { if (user_db_obj.value()) {
// user exists, let's delete // user exists, let's delete
deleteFolderRecursive(user_folder); await fs.remove(user_folder);
users_db.get('users').remove({uid: uid}).write(); users_db.get('users').remove({uid: uid}).write();
} }
res.send({success: true}); res.send({success: true});

44
backend/authentication/auth.js
View File

@@ -139,12 +139,12 @@ exports.registerUser = function(req, res) {
exports.passport.use(new LocalStrategy({ exports.passport.use(new LocalStrategy({
usernameField: 'username', usernameField: 'username',
passwordField: 'password'}, passwordField: 'password'},
function(username, password, done) { async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value(); const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); } if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); } if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) { if (user) {
return done(null, bcrypt.compareSync(password, user.passhash) ? user : false); return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
} }
} }
)); ));
@@ -226,15 +226,13 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
// change password // change password
exports.changeUserPassword = async function(user_uid, new_pass) { exports.changeUserPassword = async function(user_uid, new_pass) {
return new Promise(resolve => { try {
bcrypt.hash(new_pass, saltRounds) const hash = await bcrypt.hash(new_pass, saltRounds);
.then(function(hash) {
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write(); users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
resolve(true); return true;
}).catch(err => { } catch (err) {
resolve(false); return false;
}); }
});
} }
// change user permissions // change user permissions
@@ -352,7 +350,7 @@ exports.registerUserFile = function(user_uid, file_object, type) {
.write(); .write();
} }
exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = false) { exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) {
let success = false; let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value();
if (file_obj) { if (file_obj) {
@@ -375,20 +373,20 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
.remove({ .remove({
uid: file_uid uid: file_uid
}).write(); }).write();
if (fs.existsSync(full_path)) { if (await fs.pathExists(full_path)) {
// remove json and file // remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json'); const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json'); const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null; let youtube_id = null;
if (fs.existsSync(json_path)) { if (await fs.pathExists(json_path)) {
youtube_id = fs.readJSONSync(json_path).id; youtube_id = await fs.readJSON(json_path).id;
fs.unlinkSync(json_path); await fs.unlink(json_path);
} else if (fs.existsSync(alternate_json_path)) { } else if (await fs.pathExists(alternate_json_path)) {
youtube_id = fs.readJSONSync(alternate_json_path).id; youtube_id = await fs.readJSON(alternate_json_path).id;
fs.unlinkSync(alternate_json_path); await fs.unlink(alternate_json_path);
} }
fs.unlinkSync(full_path); await fs.unlink(full_path);
// do archive stuff // do archive stuff
@@ -397,17 +395,17 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`); const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist // use subscriptions API to remove video from the archive file, and write it to the blacklist
if (fs.existsSync(archive_path)) { if (await fs.pathExists(archive_path)) {
const line = youtube_id ? subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null; const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) { if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`); let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line // adds newline to the beginning of the line
line = '\n' + line; line = '\n' + line;
fs.appendFileSync(blacklistPath, line); await fs.appendFile(blacklistPath, line);
} }
} else { } else {
logger.info(`Could not find archive file for ${type} files. Creating...`); logger.info(`Could not find archive file for ${type} files. Creating...`);
fs.ensureFileSync(archive_path); await fs.ensureFile(archive_path);
} }
} }
} }

6
backend/db.js
View File

@@ -182,9 +182,9 @@ async function importUnregisteredFiles() {
} }
// run through check list and check each file to see if it's missing from the db // run through check list and check each file to see if it's missing from the db
dirs_to_check.forEach(dir_to_check => { for (const dir_to_check of dirs_to_check) {
// recursively get all files in dir's path // recursively get all files in dir's path
const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => { files.forEach(file => {
// check if file exists in db, if not add it // check if file exists in db, if not add it
@@ -195,7 +195,7 @@ async function importUnregisteredFiles() {
logger.verbose(`Added discovered file to the database: ${file.id}`); logger.verbose(`Added discovered file to the database: ${file.id}`);
} }
}); });
}); }
} }

86
backend/subscriptions.js
View File

@@ -79,17 +79,18 @@ async function getSubscriptionInfo(sub, user_uid = null) {
else else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
return new Promise(resolve => {
// get videos // get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1']; let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies'); let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) { if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) { if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else { } else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
} }
} }
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
if (debugMode) { if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id); logger.info('Subscribe: got info for subscription ' + sub.id);
@@ -152,7 +153,6 @@ async function getSubscriptionInfo(sub, user_uid = null) {
} }
async function unsubscribe(sub, deleteMode, user_uid = null) { async function unsubscribe(sub, deleteMode, user_uid = null) {
return new Promise(async resolve => {
let basePath = null; let basePath = null;
if (user_uid) if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
@@ -172,19 +172,17 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
} }
const appendedBasePath = getAppendedBasePath(sub, basePath); const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && fs.existsSync(appendedBasePath)) { if (deleteMode && (await fs.pathExists(appendedBasePath))) {
if (sub.archive && fs.existsSync(sub.archive)) { if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt'); const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists // deletes archive if it exists
if (fs.existsSync(archive_file_path)) { if (await fs.pathExists(archive_file_path)) {
fs.unlinkSync(archive_file_path); await fs.unlink(archive_file_path);
} }
fs.rmdirSync(sub.archive); await fs.rmdir(sub.archive);
} }
deleteFolderRecursive(appendedBasePath); await fs.remove(appendedBasePath);
} }
});
} }
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
@@ -202,7 +200,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const name = file; const name = file;
let retrievedID = null; let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write(); sub_db.get('videos').remove({uid: file_uid}).write();
return new Promise(resolve => {
let filePath = appendedBasePath; let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json'); var jsonPath = path.join(__dirname,filePath,name+'.info.json');
@@ -210,53 +208,50 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
var imageFilePath = path.join(__dirname,filePath,name+'.jpg'); var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg'); var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
jsonExists = fs.existsSync(jsonPath); const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
videoFileExists = fs.existsSync(videoFilePath); fs.pathExists(jsonPath),
imageFileExists = fs.existsSync(imageFilePath); fs.pathExists(videoFilePath),
altImageFileExists = fs.existsSync(altImageFilePath); fs.pathExists(imageFilePath),
fs.pathExists(altImageFilePath),
]);
if (jsonExists) { if (jsonExists) {
retrievedID = JSON.parse(fs.readFileSync(jsonPath, 'utf8'))['id']; retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
fs.unlinkSync(jsonPath); await fs.unlink(jsonPath);
} }
if (imageFileExists) { if (imageFileExists) {
fs.unlinkSync(imageFilePath); await fs.unlink(imageFilePath);
} }
if (altImageFileExists) { if (altImageFileExists) {
fs.unlinkSync(altImageFilePath); await fs.unlink(altImageFilePath);
} }
if (videoFileExists) { if (videoFileExists) {
fs.unlink(videoFilePath, function(err) { await fs.unlink(videoFilePath);
if (fs.existsSync(jsonPath) || fs.existsSync(videoFilePath)) { if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
resolve(false); return false;
} else { } else {
// check if the user wants the video to be redownloaded (deleteForever === false) // check if the user wants the video to be redownloaded (deleteForever === false)
if (!deleteForever && useArchive && sub.archive && retrievedID) { if (!deleteForever && useArchive && sub.archive && retrievedID) {
const archive_path = path.join(sub.archive, 'archive.txt') const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID // if archive exists, remove line with video ID
if (fs.existsSync(archive_path)) { if (await fs.pathExists(archive_path)) {
removeIDFromArchive(archive_path, retrievedID); await removeIDFromArchive(archive_path, retrievedID);
} }
} }
resolve(true); return true;
} }
});
} else { } else {
// TODO: tell user that the file didn't exist // TODO: tell user that the file didn't exist
resolve(true); return true;
} }
});
} }
async function getVideosForSub(sub, user_uid = null) { async function getVideosForSub(sub, user_uid = null) {
return new Promise(resolve => {
if (!subExists(sub.id, user_uid)) { if (!subExists(sub.id, user_uid)) {
resolve(false); return false;
return;
} }
// get sub_db // get sub_db
@@ -338,7 +333,7 @@ async function getVideosForSub(sub, user_uid = null) {
let useCookies = config_api.getConfigItem('ytdl_use_cookies'); let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) { if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) { if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else { } else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
@@ -351,6 +346,8 @@ async function getVideosForSub(sub, user_uid = null) {
// get videos // get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name); logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name); logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) { if (err && !output) {
@@ -456,23 +453,8 @@ function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
} }
// https://stackoverflow.com/a/32197381/8088021 async function removeIDFromArchive(archive_path, id) {
const deleteFolderRecursive = function(folder_to_delete) { let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (fs.existsSync(folder_to_delete)) {
fs.readdirSync(folder_to_delete).forEach((file, index) => {
const curPath = path.join(folder_to_delete, file);
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);
} else { // delete file
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(folder_to_delete);
}
};
function removeIDFromArchive(archive_path, id) {
let data = fs.readFileSync(archive_path, {encoding: 'utf-8'});
if (!data) { if (!data) {
logger.error('Archive could not be found.'); logger.error('Archive could not be found.');
return; return;
@@ -493,7 +475,7 @@ function removeIDFromArchive(archive_path, id) {
// UPDATE FILE WITH NEW DATA // UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n'); const updatedData = dataArray.join('\n');
fs.writeFileSync(archive_path, updatedData); await fs.writeFile(archive_path, updatedData);
if (line) return line; if (line) return line;
if (err) throw err; if (err) throw err;
} }

23
backend/utils.js
View File

@@ -4,6 +4,7 @@ const config_api = require('./config');
const is_windows = process.platform === 'win32'; const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) { function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path; let fixed_path = unfixed_path;
@@ -19,21 +20,21 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path; return fixed_path;
} }
function getDownloadedFilesByType(basePath, type) { async function getDownloadedFilesByType(basePath, type) {
// return empty array if the path doesn't exist // return empty array if the path doesn't exist
if (!fs.existsSync(basePath)) return []; if (!(await fs.pathExists(basePath))) return [];
let files = []; let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4'; const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = recFindByExt(basePath, ext); var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) { for (let i = 0; i < located_files.length; i++) {
let file = located_files[i]; let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length); var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
var stats = fs.statSync(file); var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONByType(type, id, basePath); var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@@ -158,17 +159,16 @@ function deleteJSONFile(name, type, customPath = null) {
} }
function recFindByExt(base,ext,files,result) async function recFindByExt(base,ext,files,result)
{ {
files = files || fs.readdirSync(base) files = files || (await fs.readdir(base))
result = result || [] result = result || []
files.forEach( for (const file of files) {
function (file) {
var newbase = path.join(base,file) var newbase = path.join(base,file)
if ( fs.statSync(newbase).isDirectory() ) if ( (await fs.stat(newbase)).isDirectory() )
{ {
result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result) result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
} }
else else
{ {
@@ -178,7 +178,6 @@ function recFindByExt(base,ext,files,result)
} }
} }
} }
)
return result return result
} }