Archive refactor to improve reliability and consistency

backend/app.js

@@ -101,7 +101,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -506,7 +505,7 @@ async function loadConfig() {
     db_api.database_initialized_bs.next(true);
 
     // creates archive path if missing
-    await fs.ensureDir(archivePath);
+    await fs.ensureDir(utils.getArchiveFolder());
 
     // check migrations
     await checkMigrations();

backend/db.js

@@ -494,8 +494,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
-        const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`);
+        const archive_path = utils.getArchiveFolder(type, uuid);
 
         // get ID from JSON
 
@@ -503,14 +502,8 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => {
         let id = null;
         if (jsonobj) id = jsonobj.id;
 
-        // use subscriptions API to remove video from the archive file, and write it to the blacklist
-        if (await fs.pathExists(archive_path)) {
-            const line = id ? await utils.removeIDFromArchive(archive_path, id) : null;
-            if (blacklistMode && line) await writeToBlacklist(type, line);
-        } else {
-            logger.info('Could not find archive file for audio files. Creating...');
-            await fs.close(await fs.open(archive_path, 'w'));
-        }
+        // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+        await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
     }
 
     if (jsonExists) await fs.unlink(jsonPath);
@@ -1110,15 +1103,3 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
     });
     return return_val;
 }
-
-// archive helper functions
-
-async function writeToBlacklist(type, line) {
-    const archivePath = path.join(__dirname, 'appdata', 'archives');
-    let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
-    // adds newline to the beginning of the line
-    line.replace('\n', '');
-    line.replace('\r', '');
-    line = '\n' + line;
-    await fs.appendFile(blacklistPath, line);
-}

backend/downloader.js

@@ -18,8 +18,6 @@ const db_api = require('./db');
 const mutex = new Mutex();
 let should_check_downloads = true;
 
-const archivePath = path.join(__dirname, 'appdata', 'archives');
-
 if (db_api.database_initialized) {
     setupDownloads();
 } else {
@@ -242,6 +240,7 @@ async function downloadQueuedFile(download_uid) {
     return new Promise(async resolve => {
         const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
         await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
 
         const url = download['url'];
@@ -249,9 +248,11 @@ async function downloadQueuedFile(download_uid) {
         const options = download['options'];
         const args = download['args'];
         const category = download['category'];
-        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+        let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
         if (options.customFileFolderPath) {
             fileFolderPath = options.customFileFolderPath;
+        } else if (download['user_uid']) {
+            fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
         }
         fs.ensureDirSync(fileFolderPath);
 
@@ -375,13 +376,19 @@ async function downloadQueuedFile(download_uid) {
 exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
     const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
     const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
 
     const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
     const globalArgs = config_api.getConfigItem('ytdl_custom_args');
     const useCookies = config_api.getConfigItem('ytdl_use_cookies');
     const is_audio = type === 'audio';
 
-    let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
+    let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
+    if (options.customFileFolderPath) {
+        fileFolderPath = options.customFileFolderPath;
+    } else if (user_uid) {
+        fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
+    }
 
     if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
 
@@ -628,6 +635,6 @@ function getArchiveFolder(fileFolderPath, options, user_uid) {
     } else if (user_uid) {
         return path.join(fileFolderPath, 'archives');
     } else {
-        return path.join(archivePath);
+        return path.join('appdata', 'archives');
     }
 }

backend/subscriptions.js

@@ -196,12 +196,11 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (!deleteForever && useArchive && sub.archive && retrievedID) {
-                const archive_path = path.join(sub.archive, 'archive.txt')
-                // if archive exists, remove line with video ID
-                if (await fs.pathExists(archive_path)) {
-                    utils.removeIDFromArchive(archive_path, retrievedID);
-                }
+            if (useArchive && retrievedID) {
+                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
+
+                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
+                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
             }
             return true;
         }
@@ -322,7 +321,7 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) {
         selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
         customFileFolderPath: getAppendedBasePath(sub, basePath),
         customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
-        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name),
+        customArchivePath: path.join(basePath, 'archives', sub.name),
         additionalArgs: sub.custom_args
     }
 

backend/test/tests.js

@@ -590,4 +590,32 @@ describe('Tasks', function() {
         const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
         assert(dummy_task_obj['data']);
     });
+});
+
+describe('Archive', async function() {
+    const archive_path = path.join('test', 'archives');
+    fs.ensureDirSync(archive_path);
+    const archive_file_path = path.join(archive_path, 'archive_video.txt');
+    const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt');
+    beforeEach(async function() {
+        if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path);
+        fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n');
+
+        if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path);
+        fs.writeFileSync(blacklist_file_path, '');
+    });
+    
+    it('Delete from archive', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false);
+        const new_archive = fs.readFileSync(archive_file_path);
+        assert(!new_archive.includes('testing2'));
+    });
+
+    it('Delete from archive - blacklist', async function() {
+        await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true);
+        const new_archive = fs.readFileSync(archive_file_path);
+        const new_blacklist = fs.readFileSync(blacklist_file_path);
+        assert(!new_archive.includes('testing2'));
+        assert(new_blacklist.includes('testing2'));
+    });
 });

backend/utils.js

@@ -218,8 +218,11 @@ function deleteJSONFile(file_path, type) {
     if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
 }
 
-async function removeIDFromArchive(archive_path, id) {
-    let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
+// archive helper functions
+
+async function removeIDFromArchive(archive_path, type, id) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    const data = await fs.readFile(archive_file, {encoding: 'utf-8'});
     if (!data) {
         logger.error('Archive could not be found.');
         return;
@@ -236,12 +239,34 @@ async function removeIDFromArchive(archive_path, id) {
         }
     }
 
+    if (lastIndex === -1) return null;
+
     const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
 
     // UPDATE FILE WITH NEW DATA
     const updatedData = dataArray.join('\n');
-    await fs.writeFile(archive_path, updatedData);
-    if (line) return line;
+    await fs.writeFile(archive_file, updatedData);
+    if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line;
+}
+
+async function writeToBlacklist(archive_folder, type, line) {
+    let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt');
+    // adds newline to the beginning of the line
+    line.replace('\n', '');
+    line.replace('\r', '');
+    line = '\n' + line;
+    await fs.appendFile(blacklistPath, line);
+}
+
+async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) {
+    const archive_file = path.join(archive_path, `archive_${type}.txt`);
+    if (await fs.pathExists(archive_path)) {
+        const line = id ? await removeIDFromArchive(archive_path, type, id) : null;
+        if (blacklistMode && line) await writeToBlacklist(archive_path, type, line);
+    } else {
+        logger.info(`Could not find archive file for file ${uid}. Creating...`);
+        await fs.close(await fs.open(archive_file, 'w'));
+    }
 }
 
 function durationStringToNumber(dur_str) {
@@ -471,6 +496,25 @@ const searchObjectByString = function(o, s) {
     return o;
 }
 
+function getArchiveFolder(type, user_uid = null, sub = null) {
+    const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
+    const subsFolderPath  = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    if (user_uid) {
+        if (sub) {
+            return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name);
+        } else {
+            return path.join(usersFolderPath, user_uid, type, 'archives');
+        }
+    } else {
+        if (sub) {
+            return path.join(subsFolderPath, 'archives', sub.name);
+        } else {
+            return path.join('appdata', 'archives');
+        }
+    }
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -500,6 +544,8 @@ module.exports = {
     fixVideoMetadataPerms: fixVideoMetadataPerms,
     deleteJSONFile: deleteJSONFile,
     removeIDFromArchive: removeIDFromArchive,
+    writeToBlacklist: writeToBlacklist,
+    deleteFileFromArchive: deleteFileFromArchive,
     getDownloadedFilesByType: getDownloadedFilesByType,
     createContainerZipFile: createContainerZipFile,
     durationStringToNumber: durationStringToNumber,
@@ -516,5 +562,6 @@ module.exports = {
     restartServer: restartServer,
     injectArgs: injectArgs,
     searchObjectByString: searchObjectByString,
+    getArchiveFolder: getArchiveFolder,
     File: File
 }

src/app/components/unified-file-card/unified-file-card.component.html

@@ -34,7 +34,7 @@
                 <mat-icon>restore</mat-icon><ng-container i18n="Delete and redownload subscription video button">Delete and redownload</ng-container>
             </button>
             <button *ngIf="file_obj.sub_id && use_youtubedl_archive" (click)="emitDeleteFile(true)" mat-menu-item>
-                <mat-icon>delete_forever</mat-icon><ng-container i18n="Delete forever subscription video button">Delete forever</ng-container>
+                <mat-icon>delete_forever</mat-icon><ng-container i18n="Delete forever subscription video button">Delete and blacklist</ng-container>
             </button>
             <button *ngIf="!file_obj.sub_id" (click)="emitDeleteFile()" mat-menu-item><mat-icon>delete</mat-icon><ng-container i18n="Delete video button">Delete</ng-container></button>
             <button *ngIf="!file_obj.sub_id && use_youtubedl_archive" (click)="emitDeleteFile(true)" mat-menu-item><mat-icon>delete_forever</mat-icon><ng-container i18n="Delete and blacklist video button">Delete and blacklist</ng-container></button>