From 8058b743eb8ca8206c8e4ed07191cdcad3a89bb5 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 18 Dec 2020 18:31:23 -0500 Subject: [PATCH] Added support for redownloading fresh uploads, which will eventually be hidden behind an opt-in setting --- backend/app.js | 66 +------------ backend/db.js | 2 +- backend/subscriptions.js | 196 ++++++++++++++++++++++++++------------- backend/utils.js | 15 ++- 4 files changed, 145 insertions(+), 134 deletions(-) diff --git a/backend/app.js b/backend/app.js index 93149d6..6476136 100644 --- a/backend/app.js +++ b/backend/app.js @@ -218,6 +218,8 @@ async function checkMigrations() { let success = await simplifyDBFileStructure(); success = success && await addMetadataPropertyToDB('view_count'); success = success && await addMetadataPropertyToDB('description'); + success = success && await addMetadataPropertyToDB('height'); + success = success && await addMetadataPropertyToDB('abr'); if (success) { logger.info('4.1->4.2+ migration complete!'); } else { logger.error('Migration failed: 4.1->4.2+'); } } @@ -225,7 +227,6 @@ async function checkMigrations() { return true; } -/* async function runFilesToDBMigration() { try { let mp3s = await getMp3s(); @@ -257,7 +258,6 @@ async function runFilesToDBMigration() { return false; } } -*/ async function simplifyDBFileStructure() { let users = users_db.get('users').value(); @@ -760,64 +760,6 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -async function getMp3s() { - let mp3s = []; - var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(audioFolderPath.length, file.length); - - var stats = await fs.stat(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp3(id, audioFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = true; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp3s.push(file_obj); - } - return mp3s; -} - -async function getMp4s(relative_path = true) { - let mp4s = []; - var files = await utils.recFindByExt(videoFolderPath, 'mp4'); - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(videoFolderPath.length, file.length); - - var stats = fs.statSync(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = await utils.getJSONMp4(id, videoFolderPath); - if (!jsonobj) continue; - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - - var size = stats.size; - - var isaudio = false; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - mp4s.push(file_obj); - } - return mp4s; -} - function getThumbnailMp3(name) { var obj = utils.getJSONMp3(name, audioFolderPath); @@ -2446,7 +2388,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { var size = stats.size; var isaudio = false; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr); parsed_files.push(file_obj); } } else { @@ -2468,7 +2410,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { if (subscription.videos) { for (let i = 0; i < subscription.videos.length; i++) { const video = subscription.videos[i]; - parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date)); + parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr)); } } res.send({ diff --git a/backend/db.js b/backend/db.js index e609a30..b7298d0 100644 --- a/backend/db.js +++ b/backend/db.js @@ -95,7 +95,7 @@ function generateFileObject(id, type, customPath = null, sub = null) { var duration = jsonobj.duration; var isaudio = type === 'audio'; var description = jsonobj.description; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); return file_obj; } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 9ab7542..7ce2a96 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -273,10 +273,7 @@ async function getVideosForSub(sub, user_uid = null) { else basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - - let appendedBasePath = null - appendedBasePath = getAppendedBasePath(sub, basePath); + let appendedBasePath = getAppendedBasePath(sub, basePath); let multiUserMode = null; if (user_uid) { @@ -286,14 +283,87 @@ async function getVideosForSub(sub, user_uid = null) { } } - const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' + const downloadConfig = await generateArgsForSubscription(sub, user_uid); + + // get videos + logger.verbose('Subscription: getting videos for subscription ' + sub.name); + + return new Promise(resolve => { + youtubedl.exec(sub.url, downloadConfig, {}, async function(err, output) { + logger.verbose('Subscription: finished check for ' + sub.name); + if (err && !output) { + logger.error(err.stderr ? err.stderr : err.message); + if (err.stderr.includes('This video is unavailable')) { + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + const outputs = err.stdout.split(/\r\n|\r|\n/); + for (let i = 0; i < outputs.length; i++) { + const output = JSON.parse(outputs[i]); + handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) + if (err.stderr.includes(output['id']) && archive_path) { + // we found a video that errored! add it to the archive to prevent future errors + if (sub.archive) { + archive_dir = sub.archive; + archive_path = path.join(archive_dir, 'archive.txt') + fs.appendFileSync(archive_path, output['id']); + } + } + } + } catch(e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + } + } + resolve(false); + } else if (output) { + if (output.length === 0 || (output.length === 1 && output[0] === '')) { + logger.verbose('No additional videos to download for ' + sub.name); + resolve(true); + } + for (let i = 0; i < output.length; i++) { + let output_json = null; + try { + output_json = JSON.parse(output[i]); + } catch(e) { + output_json = null; + } + if (!output_json) { + continue; + } + + const reset_videos = i === 0; + handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); + await setFreshUploads(sub, user_uid); + checkVideosForFreshUploads(sub, user_uid); + } + resolve(true); + } + }); + }, err => { + logger.error(err); + }); +} + +async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) { + // get basePath + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + + let appendedBasePath = getAppendedBasePath(sub, basePath); let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`; - if (sub.custom_output) { + if (desired_path) { + fullOutput = `${desired_path}.%(ext)s`; + } else if (sub.custom_output) { fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`; } - let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json']; + let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json']; let qualityPath = null; if (sub.type && sub.type === 'audio') { @@ -320,7 +390,7 @@ async function getVideosForSub(sub, user_uid = null) { let archive_dir = null; let archive_path = null; - if (useArchive) { + if (useArchive && !redownload) { if (sub.archive) { archive_dir = sub.archive; archive_path = path.join(archive_dir, 'archive.txt') @@ -350,60 +420,7 @@ async function getVideosForSub(sub, user_uid = null) { downloadConfig.push('--write-thumbnail'); } - // get videos - logger.verbose('Subscription: getting videos for subscription ' + sub.name); - - return new Promise(resolve => { - youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { - logger.verbose('Subscription: finished check for ' + sub.name); - if (err && !output) { - logger.error(err.stderr ? err.stderr : err.message); - if (err.stderr.includes('This video is unavailable')) { - logger.info('An error was encountered with at least one video, backup method will be used.') - try { - const outputs = err.stdout.split(/\r\n|\r|\n/); - for (let i = 0; i < outputs.length; i++) { - const output = JSON.parse(outputs[i]); - handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) - if (err.stderr.includes(output['id']) && archive_path) { - // we found a video that errored! add it to the archive to prevent future errors - fs.appendFileSync(archive_path, output['id']); - } - } - } catch(e) { - logger.error('Backup method failed. See error below:'); - logger.error(e); - } - } - resolve(false); - } else if (output) { - if (output.length === 0 || (output.length === 1 && output[0] === '')) { - logger.verbose('No additional videos to download for ' + sub.name); - resolve(true); - } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - - const reset_videos = i === 0; - handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); - - // TODO: Potentially store downloaded files in db? - - } - resolve(true); - } - }); - }, err => { - logger.error(err); - }); + return downloadConfig; } function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { @@ -418,6 +435,14 @@ function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_ // add to db sub_db.get('videos').push(output_json).write(); } else { + path_object = path.parse(output_json['_filename']); + const path_string = path.format(path_object); + + if (sub_db.get('videos').find({path: path_string}).value()) { + // file already exists in DB, return early to avoid reseting the download date + return; + } + db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); const url = output_json['webpage_url']; if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 @@ -468,6 +493,53 @@ function subExists(subID, user_uid = null) { return !!db.get('subscriptions').find({id: subID}).value(); } +async function setFreshUploads(sub, user_uid) { + const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); + sub.videos.forEach(async video => { + if (current_date === video['upload_date'].replace(/-/g, '')) { + // set upload as fresh + const video_uid = video['uid']; + await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']); + } + }); +} + +async function checkVideosForFreshUploads(sub, user_uid) { + const current_date = new Date().toISOString().split('T')[0].replace(/-/g, ''); + sub.videos.forEach(async video => { + if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) { + checkVideoIfBetterExists(video, sub, user_uid) + } + }); +} + +async function checkVideoIfBetterExists(file_obj, sub, user_uid) { + const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4); + const downloadConfig = generateArgsForSubscription(sub, user_uid, true, new_path); + logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); + // simulate a download to verify that a better version exists + youtubedl.getInfo(file_obj['url'], downloadConfig, (err, output) => { + if (err) { + // video is not available anymore for whatever reason + } else if (output) { + console.log(output); + const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; + if (output[metric_to_compare] > file_obj[metric_to_compare]) { + // download new video as the simulated one is better + youtubedl.exec(file_obj['url'], downloadConfig, async (err, output) => { + if (err) { + logger.verbose(`Failed to download better version of video ${file_obj['id']}`); + } else if (output) { + logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); + await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']); + } + }); + } + } + }); + await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']); +} + // helper functions function getAppendedBasePath(sub, base_path) { diff --git a/backend/utils.js b/backend/utils.js index 988a112..b18ed6a 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -41,18 +41,12 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) { files.push(jsonobj); continue; } - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; var upload_date = jsonobj.upload_date; upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - - var size = stats.size; var isaudio = type === 'audio'; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new File(id, jsonobj.title, jsonobj.thumbnail, isaudio, jsonobj.duration, jsonobj.webpage_url, jsonobj.uploader, + stats.size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr); files.push(file_obj); } return files; @@ -189,7 +183,7 @@ async function recFindByExt(base,ext,files,result) // objects -function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description) { +function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { this.id = id; this.title = title; this.thumbnailURL = thumbnailURL; @@ -201,6 +195,9 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p this.path = path; this.upload_date = upload_date; this.description = description; + this.view_count = view_count; + this.height = height; + this.abr = abr; } module.exports = {