TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-05 18:51:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

If multiple videos exist in one URL, a playlist will be auto generated

Browse Source 
Removed tomp3 and tomp4 routes, replaced with /downloadFile

Simplified category->playlist conversion

Simplified playlist creation

Simplified file deletion

Playlist duration calculation is now done on the backend (categories uses this now too)

removeIDFromArchive moved from subscriptions->utils

Added plumbing to support type agnostic playlists
This commit is contained in:
Isaac Abadi
2021-05-30 00:39:00 -06:00
parent e2c31319cf
commit 4ea239170e
15 changed files with 381 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
backend/app.js
View File

@@ -231,7 +231,7 @@ async function runFilesToDBMigration() {
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
await db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
}
@@ -240,7 +240,7 @@ async function runFilesToDBMigration() {
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
await db_api.registerFileDB(file_obj.id + '.mp4', 'video');
db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
}
@@ -837,87 +837,6 @@ function getVideoFormatID(name)
}
}
// TODO: add to db_api and support multi-user mode
async function deleteFile(uid, uuid = null, blacklistMode = false) {
const file_obj = await db_api.getVideo(uid, uuid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[name]) {
try {
for (let i = 0; i < config_api.descriptors[name].length; i++) {
config_api.descriptors[name][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = path.join(archivePath, `archive_${type}.txt`);
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null;
if (jsonobj) id = jsonobj.id;
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = id ? await subscriptions_api.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) await writeToBlacklist(type, line);
} else {
logger.info('Could not find archive file for audio files. Creating...');
await fs.close(await fs.open(archive_path, 'w'));
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
/**
* @param {'audio' | 'video'} type
* @param {string[]} fileNames
@@ -1036,7 +955,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
download['downloading'] = false;
download['timestamp_end'] = Date.now();
var file_uid = null;
var file_objs = [];
let new_date = Date.now();
let difference = (new_date - date)/1000;
logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
@@ -1108,9 +1027,12 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
}
// registers file in DB
file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings);
const file_obj = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath, category, options.cropFileSettings);
// TODO: remove the following line
if (file_name) file_names.push(file_name);
file_objs.push(file_obj);
}
let is_playlist = file_names.length > 1;
@@ -1126,12 +1048,22 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
download['fileNames'] = is_playlist ? file_names : [full_file_path]
updateDownloads();
var videopathEncoded = encodeURIComponent(file_names[0]);
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, file_objs[0]['thumbnailURL'], options.user);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
logger.error('Downloaded file failed to result in metadata object.');
}
resolve({
[(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded,
file_names: is_playlist ? file_names : null,
uid: file_uid
file_uids: file_objs.map(file_obj => file_obj.uid),
container: container
});
}
});
@@ -1260,7 +1192,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) {
videopathEncoded = encodeURIComponent(utils.removeFileExtension(base_file_name));
resolve({
[is_audio ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded,
encodedPath: videopathEncoded,
file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready
uid: file_uid
});
@@ -1727,18 +1659,18 @@ app.use(function(req, res, next) {
app.use(compression());
const optionalJwt = function (req, res, next) {
const optionalJwt = async function (req, res, next) {
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
if (multiUserMode && ((req.body && req.body.uuid) || (req.query && req.query.uuid)) && (req.path.includes('/api/getFile') ||
req.path.includes('/api/stream') ||
req.path.includes('/api/getPlaylist') ||
req.path.includes('/api/downloadFile'))) {
req.path.includes('/api/downloadFileFromServer'))) {
// check if shared video
const using_body = req.body && req.body.uuid;
const uuid = using_body ? req.body.uuid : req.query.uuid;
const uid = using_body ? req.body.uid : req.query.uid;
const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : db_api.getPlaylist(playlist_id, uuid, true);
const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true);
if (file) {
req.can_watch = true;
return next();
@@ -1783,38 +1715,10 @@ app.post('/api/restartServer', optionalJwt, (req, res) => {
res.send({success: true});
});
app.post('/api/tomp3', optionalJwt, async function(req, res) {
var url = req.body.url;
var options = {
customArgs: req.body.customArgs,
customOutput: req.body.customOutput,
maxBitrate: req.body.maxBitrate,
customQualityConfiguration: req.body.customQualityConfiguration,
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
ui_uid: req.body.ui_uid,
user: req.isAuthenticated() ? req.user.uid : null
}
const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override') || config_api.globalArgsRequiresSafeDownload();
if (safeDownloadOverride) logger.verbose('Download is running with the safe download override.');
const is_playlist = url.includes('playlist');
let result_obj = null;
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
if (result_obj) {
res.send(result_obj);
} else {
res.sendStatus(500);
}
});
app.post('/api/tomp4', optionalJwt, async function(req, res) {
app.post('/api/downloadFile', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos
var url = req.body.url;
const url = req.body.url;
const type = req.body.type;
var options = {
customArgs: req.body.customArgs,
customOutput: req.body.customOutput,
@@ -1833,7 +1737,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
let result_obj = null;
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
result_obj = await downloadFileByURL_exec(url, type, options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
if (result_obj) {
@@ -1936,43 +1840,22 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
// these are returned
let files = null;
let playlists = null;
const uuid = req.isAuthenticated() ? req.user.uid : null;
let subscriptions = config_api.getConfigItem('ytdl_allow_subscriptions') ? (subscriptions_api.getSubscriptions(req.isAuthenticated() ? req.user.uid : null)) : [];
// get basic info depending on multi-user mode being enabled
if (req.isAuthenticated()) {
if (uuid) {
files = auth_api.getUserVideos(req.user.uid);
playlists = auth_api.getUserPlaylists(req.user.uid, files);
} else {
files = db.get('files').value();
playlists = JSON.parse(JSON.stringify(db.get('playlists').value()));
const categories = db.get('categories').value();
if (categories) {
categories.forEach(category => {
const audio_files = files && files.filter(file => file.category && file.category.uid === category.uid && file.isAudio);
const video_files = files && files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio);
if (audio_files && audio_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: audio_files[0].thumbnailURL,
thumbnailPath: audio_files[0].thumbnailPath,
fileNames: audio_files.map(file => file.id),
type: 'audio',
auto: true
});
}
if (video_files && video_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: video_files[0].thumbnailURL,
thumbnailPath: video_files[0].thumbnailPath,
fileNames: video_files.map(file => file.id),
type: 'video',
auto: true
});
}
});
}
}
const categories = categories_api.getCategoriesAsPlaylists(files);
if (categories) {
playlists = playlists.concat(categories);
}
// loop through subscriptions and add videos
@@ -2439,26 +2322,8 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
let uids = req.body.uids;
let type = req.body.type;
let thumbnailURL = req.body.thumbnailURL;
let duration = req.body.duration;
let new_playlist = {
name: playlistName,
uids: uids,
id: shortid.generate(),
thumbnailURL: thumbnailURL,
type: type,
registered: Date.now(),
duration: duration
};
if (req.isAuthenticated()) {
auth_api.addPlaylist(req.user.uid, new_playlist, type);
} else {
db.get(`playlists`)
.push(new_playlist)
.write();
}
const new_playlist = await db_api.createPlaylist(playlistName, uids, type, thumbnailURL, req.isAuthenticated() ? req.user.uid : null);
res.send({
new_playlist: new_playlist,
@@ -2517,7 +2382,7 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => {
app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
let playlist = req.body.playlist;
let success = db_api.updatePlaylist(playlist, req.user && req.user.uid);
let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid);
res.send({
success: success
});
@@ -2551,20 +2416,14 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
app.post('/api/deleteFile', optionalJwt, async (req, res) => {
const uid = req.body.uid;
const blacklistMode = req.body.blacklistMode;
if (req.isAuthenticated()) {
let success = await auth_api.deleteUserFile(req.user.uid, uid, blacklistMode);
res.send(success);
return;
}
const uuid = req.isAuthenticated() ? req.user.uid : null;
let wasDeleted = false;
wasDeleted = await deleteFile(uid, null, blacklistMode);
db.get('files').remove({uid: uid}).write();
wasDeleted = await db_api.deleteFile(uid, uuid, blacklistMode);
res.send(wasDeleted);
});
app.post('/api/downloadFile', optionalJwt, async (req, res) => {
app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
let uid = req.body.uid;
let uuid = req.body.uuid;
let playlist_id = req.body.playlist_id;

109
backend/authentication/auth.js
View File

@@ -1,12 +1,10 @@
const path = require('path');
const config_api = require('../config');
const consts = require('../consts');
var subscriptions_api = require('../subscriptions')
const fs = require('fs-extra');
var jwt = require('jsonwebtoken');
const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4');
var bcrypt = require('bcryptjs');
const bcrypt = require('bcryptjs');
var LocalStrategy = require('passport-local').Strategy;
var LdapStrategy = require('passport-ldapauth');
@@ -299,11 +297,6 @@ exports.getUserVideo = function(user_uid, file_uid, requireSharing = false) {
return file;
}
exports.addPlaylist = function(user_uid, new_playlist) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
return true;
}
exports.updatePlaylistFiles = function(user_uid, playlistID, new_filenames) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID}).assign({fileNames: new_filenames});
return true;
@@ -317,35 +310,6 @@ exports.removePlaylist = function(user_uid, playlistID) {
exports.getUserPlaylists = function(user_uid, user_files = null) {
const user = users_db.get('users').find({uid: user_uid}).value();
const playlists = JSON.parse(JSON.stringify(user['playlists']));
const categories = db.get('categories').value();
if (categories && user_files) {
categories.forEach(category => {
const audio_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && file.isAudio);
const video_files = user_files && user_files.filter(file => file.category && file.category.uid === category.uid && !file.isAudio);
if (audio_files && audio_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: audio_files[0].thumbnailURL,
thumbnailPath: audio_files[0].thumbnailPath,
fileNames: audio_files.map(file => file.id),
type: 'audio',
uid: user_uid,
auto: true
});
}
if (video_files && video_files.length > 0) {
playlists.push({
name: category['name'],
thumbnailURL: video_files[0].thumbnailURL,
thumbnailPath: video_files[0].thumbnailPath,
fileNames: video_files.map(file => file.id),
type: 'video',
uid: user_uid,
auto: true
});
}
});
}
return playlists;
}
@@ -369,75 +333,6 @@ exports.registerUserFile = function(user_uid, file_object) {
.write();
}
exports.deleteUserFile = async function(user_uid, file_uid, blacklistMode = false) {
let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files`).find({uid: file_uid}).value();
if (file_obj) {
const type = file_obj.isAudio ? 'audio' : 'video';
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
// close descriptors
if (config_api.descriptors[file_obj.id]) {
try {
for (let i = 0; i < config_api.descriptors[file_obj.id].length; i++) {
config_api.descriptors[file_obj.id][i].destroy();
}
} catch(e) {
}
}
const full_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext);
users_db.get('users').find({uid: user_uid}).get(`files`)
.remove({
uid: file_uid
}).write();
if (await fs.pathExists(full_path)) {
// remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null;
if (await fs.pathExists(json_path)) {
youtube_id = await fs.readJSON(json_path).id;
await fs.unlink(json_path);
} else if (await fs.pathExists(alternate_json_path)) {
youtube_id = await fs.readJSON(alternate_json_path).id;
await fs.unlink(alternate_json_path);
}
await fs.unlink(full_path);
// do archive stuff
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line
line = '\n' + line;
await fs.appendFile(blacklistPath, line);
}
} else {
logger.info(`Could not find archive file for ${type} files. Creating...`);
await fs.ensureFile(archive_path);
}
}
}
success = true;
} else {
success = false;
logger.warn(`User file ${file_uid} does not exist!`);
}
return success;
}
exports.changeSharingMode = function(user_uid, file_uid, is_playlist, enabled) {
let success = false;
const user_db_obj = users_db.get('users').find({uid: user_uid});

21
backend/categories.js
View File

@@ -1,4 +1,5 @@
const config_api = require('./config');
const utils = require('./utils');
var logger = null;
var db = null;
@@ -68,6 +69,24 @@ function getCategories() {
return categories ? categories : null;
}
function getCategoriesAsPlaylists(files = null) {
const categories_as_playlists = [];
const available_categories = getCategories();
if (available_categories && files) {
for (category of available_categories) {
const files_that_match = utils.addUIDsToCategory(category, files);
if (files_that_match && files_that_match.length > 0) {
category['thumbnailURL'] = files_that_match[0].thumbnailURL;
category['thumbnailPath'] = files_that_match[0].thumbnailPath;
category['duration'] = files_that_match.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
category['id'] = category['uid'];
categories_as_playlists.push(category);
}
}
}
return categories_as_playlists;
}
function applyCategoryRules(file_json, rules, category_name) {
let rules_apply = false;
for (let i = 0; i < rules.length; i++) {
@@ -126,4 +145,6 @@ async function addTagToExistingTags(tag) {
module.exports = {
initialize: initialize,
categorize: categorize,
getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists
}

155
backend/db.js
View File

@@ -53,14 +53,14 @@ exports.registerFileDB = (file_path, type, multiUserMode = null, sub = null, cus
}
}
const file_uid = registerFileDBManual(db_path, file_object);
const file_obj = registerFileDBManual(db_path, file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
}
return file_uid;
return file_obj;
}
function registerFileDBManual(db_path, file_object) {
@@ -75,7 +75,7 @@ function registerFileDBManual(db_path, file_object) {
// add new file to db
db_path.push(file_object).write();
return file_object['uid'];
return file_object;
}
function generateFileObject(id, type, customPath = null, sub = null) {
@@ -224,17 +224,47 @@ exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) =>
return preimported_file_paths;
}
exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => {
let new_playlist = {
name: playlist_name,
uids: uids,
id: uuid(),
thumbnailURL: thumbnail_url,
type: type,
registered: Date.now(),
};
const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid);
new_playlist.duration = duration;
if (user_uid) {
users_db.get('users').find({uid: user_uid}).get(`playlists`).push(new_playlist).write();
} else {
db.get(`playlists`)
.push(new_playlist)
.write();
}
return new_playlist;
}
exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => {
let playlist = null
if (user_uid) {
playlist = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlist_id}).value();
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
} else {
playlist = db.get(`playlists`).find({id: playlist_id}).value();
}
if (!playlist) {
playlist = db.get('categories').find({uid: playlist_id}).value();
if (playlist) {
// category found
const files = await exports.getFiles(user_uid);
utils.addUIDsToCategory(playlist, files);
}
}
// converts playlists to new UID-based schema
if (playlist && playlist['fileNames'] && !playlist['uids']) {
playlist['uids'] = [];
@@ -248,11 +278,18 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
exports.updatePlaylist(playlist, user_uid);
}
// prevent unauthorized users from accessing the file info
if (require_sharing && !playlist['sharingEnabled']) return null;
return playlist;
}
exports.updatePlaylist = (playlist, user_uid = null) => {
exports.updatePlaylist = async (playlist, user_uid = null) => {
let playlistID = playlist.id;
const duration = await exports.calculatePlaylistDuration(playlist, user_uid);
playlist.duration = duration;
let db_loc = null;
if (user_uid) {
db_loc = users_db.get('users').find({uid: user_uid}).get(`playlists`).find({id: playlistID});
@@ -263,6 +300,103 @@ exports.updatePlaylist = (playlist, user_uid = null) => {
return true;
}
exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => {
if (!playlist_file_objs) {
playlist_file_objs = [];
for (let i = 0; i < playlist['uids'].length; i++) {
const uid = playlist['uids'][i];
const file_obj = await exports.getVideo(uid, uuid);
if (file_obj) playlist_file_objs.push(file_obj);
}
}
return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
}
exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
const file_obj = await exports.getVideo(uid, uuid);
const type = file_obj.isAudio ? 'audio' : 'video';
const folderPath = path.dirname(file_obj.path);
const ext = type === 'audio' ? 'mp3' : 'mp4';
const name = file_obj.id;
const filePathNoExtension = utils.removeFileExtension(file_obj.path);
var jsonPath = `${file_obj.path}.info.json`;
var altJSONPath = `${filePathNoExtension}.info.json`;
var thumbnailPath = `${filePathNoExtension}.webp`;
var altThumbnailPath = `${filePathNoExtension}.jpg`;
jsonPath = path.join(__dirname, jsonPath);
altJSONPath = path.join(__dirname, altJSONPath);
let jsonExists = await fs.pathExists(jsonPath);
let thumbnailExists = await fs.pathExists(thumbnailPath);
if (!jsonExists) {
if (await fs.pathExists(altJSONPath)) {
jsonExists = true;
jsonPath = altJSONPath;
}
}
if (!thumbnailExists) {
if (await fs.pathExists(altThumbnailPath)) {
thumbnailExists = true;
thumbnailPath = altThumbnailPath;
}
}
let fileExists = await fs.pathExists(file_obj.path);
if (config_api.descriptors[uid]) {
try {
for (let i = 0; i < config_api.descriptors[uid].length; i++) {
config_api.descriptors[uid][i].destroy();
}
} catch(e) {
}
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null;
if (jsonobj) id = jsonobj.id;
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (await fs.pathExists(archive_path)) {
const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
if (blacklistMode && line) await writeToBlacklist(type, line);
} else {
logger.info('Could not find archive file for audio files. Creating...');
await fs.close(await fs.open(archive_path, 'w'));
}
}
if (jsonExists) await fs.unlink(jsonPath);
if (thumbnailExists) await fs.unlink(thumbnailPath);
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
base_db_path.get('files').remove({uid: uid}).write();
if (fileExists) {
await fs.unlink(file_obj.path);
if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) {
return false;
} else {
return true;
}
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that
exports.getVideoUIDByID = (file_id, uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
@@ -270,12 +404,17 @@ exports.getVideoUIDByID = (file_id, uuid = null) => {
return file_obj ? file_obj['uid'] : null;
}
exports.getVideo = async (file_uid, uuid, sub_id) => {
exports.getVideo = async (file_uid, uuid = null, sub_id = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');
return sub_db_path.find({uid: file_uid}).value();
}
exports.getFiles = async (uuid = null) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
return base_db_path.get('files').value();
}
exports.setVideoProperty = async (file_uid, assignment_obj, uuid, sub_id) => {
const base_db_path = uuid ? users_db.get('users').find({uid: uuid}) : db;
const sub_db_path = sub_id ? base_db_path.get('subscriptions').find({id: sub_id}).get('videos') : base_db_path.get('files');

30
backend/subscriptions.js
View File

@@ -243,7 +243,7 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
utils.removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
@@ -597,33 +597,6 @@ function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
module.exports = {
getSubscription : getSubscription,
getSubscriptionByName : getSubscriptionByName,
@@ -634,7 +607,6 @@ module.exports = {
unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub,
removeIDFromArchive : removeIDFromArchive,
setLogger : setLogger,
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple

50
backend/utils.js
View File

@@ -202,6 +202,52 @@ function deleteJSONFile(name, type, customPath = null) {
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
}
let dataArray = data.split('\n'); // convert file data in an array
const searchKeyword = id; // we are looking for a line, contains, key word id in the file
let lastIndex = -1; // let say, we have not found the keyword
for (let index=0; index<dataArray.length; index++) {
if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
lastIndex = index; // found a line includes a id keyword
break;
}
}
const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}
function durationStringToNumber(dur_str) {
if (typeof dur_str === 'number') return dur_str;
let num_sum = 0;
const dur_str_parts = dur_str.split(':');
for (let i = dur_str_parts.length-1; i >= 0; i--) {
num_sum += parseInt(dur_str_parts[i])*(60**(dur_str_parts.length-1-i));
}
return num_sum;
}
function getMatchingCategoryFiles(category, files) {
return files && files.filter(file => file.category && file.category.uid === category.uid);
}
function addUIDsToCategory(category, files) {
const files_that_match = getMatchingCategoryFiles(category, files);
category['uids'] = files_that_match.map(file => file.uid);
return files_that_match;
}
async function recFindByExt(base,ext,files,result)
{
@@ -268,8 +314,12 @@ module.exports = {
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
removeIDFromArchive, removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile,
durationStringToNumber: durationStringToNumber,
getMatchingCategoryFiles: getMatchingCategoryFiles,
addUIDsToCategory: addUIDsToCategory,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
wait: wait,