From 0e5c78db0d0a87e8918623f9eb3c2eb87525edd8 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Sat, 29 Apr 2023 19:34:08 -0400 Subject: [PATCH] Modified archive logic to align with #366 --- backend/downloader.js | 4 +-- backend/subscriptions.js | 31 +++++++++---------- .../unified-file-card.component.html | 9 ++---- src/app/main/main.component.ts | 2 -- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/backend/downloader.js b/backend/downloader.js index 9850ccc..b3915fa 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -221,6 +221,7 @@ async function collectInfo(download_uid) { return; } + // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); if (useYoutubeDLArchive && !options.ignoreArchive) { const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']); @@ -386,8 +387,7 @@ async function downloadQueuedFile(download_uid) { // registers file in DB const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); - const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']); + await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']); notifications_api.sendDownloadNotification(file_obj, download['user_uid']); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 5aa28e0..a6d2b18 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -199,8 +199,13 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, return false; } else { // check if the user wants the video to be redownloaded (deleteForever === false) - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useArchive && !deleteForever) { + if (deleteForever) { + // ensure video is in the archives + const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id); + if (!exists_in_archive) { + await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id); + } + } else { await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id); } return true; @@ -364,15 +369,12 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push(...qualityPath) - // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time) - const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id); - logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`) - const archive_path = path.join(appendedBasePath, 'archive.txt'); - await fs.writeFile(archive_path, archive_text); - downloadConfig.push('--download-archive', archive_path); - } + // skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time) + const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id); + logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_text.split('\n').length - 1} entries.`) + const archive_path = path.join(appendedBasePath, 'archive.txt'); + await fs.writeFile(archive_path, archive_text); + downloadConfig.push('--download-archive', archive_path); if (sub.custom_args) { const customArgsArray = sub.custom_args.split(',,'); @@ -428,11 +430,8 @@ async function getFilesToDownload(sub, output_jsons) { logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`) continue; } - const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id); - if (exists_in_archive) continue; - } + const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id); + if (exists_in_archive) continue; files_to_download.push(output_json); } diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index 55a4cb5..5e8c938 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -35,14 +35,9 @@ - - + - + diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 5c68472..731dbe6 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -50,7 +50,6 @@ export class MainComponent implements OnInit { allowQualitySelect = false; downloadOnlyMode = false; forceAutoplay = false; - use_youtubedl_archive = false; globalCustomArgs = null; allowAdvancedDownload = false; useDefaultDownloadingAgent = true; @@ -188,7 +187,6 @@ export class MainComponent implements OnInit { && this.postsService.hasPermission('filemanager'); this.downloadOnlyMode = this.postsService.config['Extra']['download_only_mode']; this.forceAutoplay = this.postsService.config['Extra']['force_autoplay']; - this.use_youtubedl_archive = this.postsService.config['Downloader']['use_youtubedl_archive']; this.globalCustomArgs = this.postsService.config['Downloader']['custom_args']; this.youtubeSearchEnabled = this.postsService.config['API'] && this.postsService.config['API']['use_youtube_API'] && this.postsService.config['API']['youtube_API_key'];