TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-19 11:10:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added support for redownloading fresh uploads, which will eventually be hidden behind an opt-in setting

Browse Source
This commit is contained in:
Isaac Abadi
2020-12-18 18:31:23 -05:00
parent 29b8dc227c
commit 8058b743eb
4 changed files with 145 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
backend/app.js
View File

@@ -218,6 +218,8 @@ async function checkMigrations() {
let success = await simplifyDBFileStructure();
success = success && await addMetadataPropertyToDB('view_count');
success = success && await addMetadataPropertyToDB('description');
success = success && await addMetadataPropertyToDB('height');
success = success && await addMetadataPropertyToDB('abr');
if (success) { logger.info('4.1->4.2+ migration complete!'); }
else { logger.error('Migration failed: 4.1->4.2+'); }
}
@@ -225,7 +227,6 @@ async function checkMigrations() {
return true;
}
/*
async function runFilesToDBMigration() {
try {
let mp3s = await getMp3s();
@@ -257,7 +258,6 @@ async function runFilesToDBMigration() {
return false;
}
}
*/
async function simplifyDBFileStructure() {
let users = users_db.get('users').value();
@@ -760,64 +760,6 @@ function generateEnvVarConfigItem(key) {
return {key: key, value: process['env'][key]};
}
async function getMp3s() {
let mp3s = [];
var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(audioFolderPath.length, file.length);
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = await utils.getJSONMp3(id, audioFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = true;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
mp3s.push(file_obj);
}
return mp3s;
}
async function getMp4s(relative_path = true) {
let mp4s = [];
var files = await utils.recFindByExt(videoFolderPath, 'mp4');
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(videoFolderPath.length, file.length);
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = await utils.getJSONMp4(id, videoFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var size = stats.size;
var isaudio = false;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
mp4s.push(file_obj);
}
return mp4s;
}
function getThumbnailMp3(name)
{
var obj = utils.getJSONMp3(name, audioFolderPath);
@@ -2446,7 +2388,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
var size = stats.size;
var isaudio = false;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
parsed_files.push(file_obj);
}
} else {
@@ -2468,7 +2410,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
if (subscription.videos) {
for (let i = 0; i < subscription.videos.length; i++) {
const video = subscription.videos[i];
parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date));
parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
}
}
res.send({

2
backend/db.js
View File

@@ -95,7 +95,7 @@ function generateFileObject(id, type, customPath = null, sub = null) {
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description);
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}

196
backend/subscriptions.js
View File

@@ -273,10 +273,7 @@ async function getVideosForSub(sub, user_uid = null) {
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = null
appendedBasePath = getAppendedBasePath(sub, basePath);
let appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null;
if (user_uid) {
@@ -286,14 +283,87 @@ async function getVideosForSub(sub, user_uid = null) {
}
}
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
fs.appendFileSync(archive_path, output['id']);
}
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name);
resolve(true);
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
}
resolve(true);
}
});
}, err => {
logger.error(err);
});
}
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
// get basePath
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = getAppendedBasePath(sub, basePath);
let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
if (sub.custom_output) {
if (desired_path) {
fullOutput = `${desired_path}.%(ext)s`;
} else if (sub.custom_output) {
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
@@ -320,7 +390,7 @@ async function getVideosForSub(sub, user_uid = null) {
let archive_dir = null;
let archive_path = null;
if (useArchive) {
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
@@ -350,60 +420,7 @@ async function getVideosForSub(sub, user_uid = null) {
downloadConfig.push('--write-thumbnail');
}
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr ? err.stderr : err.message);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
fs.appendFileSync(archive_path, output['id']);
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name);
resolve(true);
}
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos);
// TODO: Potentially store downloaded files in db?
}
resolve(true);
}
});
}, err => {
logger.error(err);
});
return downloadConfig;
}
function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) {
@@ -418,6 +435,14 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_
// add to db
sub_db.get('videos').push(output_json).write();
} else {
path_object = path.parse(output_json['_filename']);
const path_string = path.format(path_object);
if (sub_db.get('videos').find({path: path_string}).value()) {
// file already exists in DB, return early to avoid reseting the download date
return;
}
db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub);
const url = output_json['webpage_url'];
if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
@@ -468,6 +493,53 @@ function subExists(subID, user_uid = null) {
return !!db.get('subscriptions').find({id: subID}).value();
}
async function setFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (current_date === video['upload_date'].replace(/-/g, '')) {
// set upload as fresh
const video_uid = video['uid'];
await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
}
});
}
async function checkVideosForFreshUploads(sub, user_uid) {
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
sub.videos.forEach(async video => {
if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
checkVideoIfBetterExists(video, sub, user_uid)
}
});
}
async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
const downloadConfig = generateArgsForSubscription(sub, user_uid, true, new_path);
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
// simulate a download to verify that a better version exists
youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => {
if (err) {
// video is not available anymore for whatever reason
} else if (output) {
console.log(output);
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
if (output[metric_to_compare] > file_obj[metric_to_compare]) {
// download new video as the simulated one is better
youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => {
if (err) {
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
} else if (output) {
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
}
});
}
}
});
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
}
// helper functions
function getAppendedBasePath(sub, base_path) {

15
backend/utils.js
View File

@@ -41,18 +41,12 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
files.push(jsonobj);
continue;
}
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var size = stats.size;
var isaudio = type === 'audio';
var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader,
stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
files.push(file_obj);
}
return files;
@@ -189,7 +183,7 @@ async function recFindByExt(base,ext,files,result)
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description) {
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
this.id = id;
this.title = title;
this.thumbnailURL = thumbnailURL;
@@ -201,6 +195,9 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
this.path = path;
this.upload_date = upload_date;
this.description = description;
this.view_count = view_count;
this.height = height;
this.abr = abr;
}
module.exports = {