Implemented basic db-based archive functionality, converted old archive functionality to new system

Public API v1.yaml

@@ -1632,6 +1632,9 @@ components:
           $ref: '#/components/schemas/FileType'
         cropFileSettings:
           $ref: '#/components/schemas/CropFileSettings'
+        ignoreArchive:
+          type: boolean
+          description: If using youtube-dl archive, download will ignore it
     DownloadResponse:
       type: object
       properties:
@@ -2080,17 +2083,12 @@ components:
         type:
           $ref: '#/components/schemas/FileType'
     DownloadArchiveRequest:
-      required:
-        - sub
       type: object
       properties:
-        sub:
+        type:
-          required:
+          $ref: '#/components/schemas/FileType'
-            - archive_dir
+        sub_id:
-          type: object
+          type: string
-          properties:
-            archive_dir:
-              type: string
     UpdaterStatus:
       required:
         - details

backend/app.js

@@ -522,9 +522,6 @@ async function loadConfig() {
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
-    // creates archive path if missing
-    await fs.ensureDir(utils.getArchiveFolder());
-
     // check migrations
     await checkMigrations();
 
@@ -830,7 +827,8 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
         youtubeUsername: req.body.youtubeUsername,
         youtubePassword: req.body.youtubePassword,
         ui_uid: req.body.ui_uid,
-        cropFileSettings: req.body.cropFileSettings
+        cropFileSettings: req.body.cropFileSettings,
+        ignoreArchive: req.body.ignoreArchive
     };
 
     const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -1517,15 +1515,18 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
 });
 
 app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
-    let sub = req.body.sub;
+    const uuid = req.isAuthenticated() ? req.user.uid : null;
-    let archive_dir = sub.archive;
+    const sub_id = req.body.sub_id; 
+    const type = req.body.type;
 
-    let full_archive_path = path.join(archive_dir, 'archive.txt');
+    const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
 
-    if (await fs.pathExists(full_archive_path)) {
+    if (archive_text !== null && archive_text !== undefined) {
-        res.sendFile(full_archive_path);
+        res.setHeader('Content-type', "application/octet-stream");
+        res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
+        res.send(archive_text);
     } else {
-        res.sendStatus(404);
+        res.sendStatus(400);
     }
 
 });

backend/archive.js

@@ -3,20 +3,27 @@ const fs = require('fs-extra');
 
 const db_api = require('./db');
 
-exports.generateArchive = async (user_uid = null, sub_id = null) => {
+exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
-    const archive_items = await db_api.getRecords('archives', {user_uid: user_uid, sub_id: sub_id});
+    const filter = {user_uid: user_uid, sub_id: sub_id};
+    if (type) filter['type'] = type;
+    const archive_items = await db_api.getRecords('archives', filter);
     const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
     return archive_item_lines.join('\n');
 }
 
 exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
     const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
-    const success = await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}, type: type});
+    const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
+    return success;
+}
+
+exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
+    const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
     return success;
 }
 
 exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
-    const archive_item = await db_api.getRecord('archives', {'key.extractor': extractor, 'key.id': id, type: type, user_uid: user_uid, sub_id: sub_id});
+    const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
     return !!archive_item;
 }
 
@@ -37,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id =
         // we can't do a bulk write because we need to avoid duplicate archive items existing in db
 
         const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
-        await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}});
+        await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id});
         archive_import_count++;
     }
     return archive_import_count;
@@ -71,10 +78,8 @@ exports.importArchives = async () => {
 
 const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => {
     return {
-        key: {
+        extractor: extractor,
-            extractor: extractor,
+        id: id,
-            id: id
-        },
         type: type,
         user_uid: user_uid ? user_uid : null,
         sub_id: sub_id ? sub_id : null

backend/db.js

@@ -64,8 +64,7 @@ const tables = {
         primary_key: 'uid'
     },
     archives: {
-        name: 'archives',
+        name: 'archives'
-        primary_key: 'key'
     },
     test: {
         name: 'test'
@@ -510,16 +509,22 @@ exports.deleteFile = async (uid, blacklistMode = false) => {
 
     let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     if (useYoutubeDLArchive) {
-        const archive_path = utils.getArchiveFolder(type, file_obj.user_uid, file_obj.sub_id ? (await exports.getRecord('subscriptions', {id: file_obj.sub_id})) : null);
+        // get id/extractor from JSON
 
-        // get ID from JSON
+        const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
-
+        let retrievedID = null;
-        var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
+        let retrievedExtractor = null;
-        let id = null;
+        if (info_json) {
-        if (jsonobj) id = jsonobj.id;
+            retrievedID = info_json['id'];
+            retrievedExtractor = info_json['extractor'];
+        }
 
         // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-        await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
+        if (!blacklistMode) {
+            // workaround until a files_api is created (using archive_api would make a circular dependency)
+            await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
+            // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
+        }
     }
 
     if (jsonExists) await fs.unlink(jsonPath);

backend/downloader.js

@@ -15,6 +15,7 @@ const categories_api = require('./categories');
 const utils = require('./utils');
 const db_api = require('./db');
 const notifications_api = require('./notifications');
+const archive_api = require('./archive');
 
 const mutex = new Mutex();
 let should_check_downloads = true;
@@ -202,6 +203,20 @@ async function collectInfo(download_uid) {
         return;
     }
 
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive && !options.ignoreArchive) {
+        const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
+        if (exists_in_archive) {
+            const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
+            logger.warn(error);
+            if (download_uid) {
+                const download = await db_api.getRecord('download_queue', {uid: download_uid});
+                await handleDownloadError(download, error);
+                return;
+            }
+        }
+    }
+
     let category = null;
 
     // check if it fits into a category. If so, then get info again using new args
@@ -353,19 +368,14 @@ async function downloadQueuedFile(download_uid) {
                     // registers file in DB
                     const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
+                    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+                    if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']);
+
                     notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
 
                     file_objs.push(file_obj);
                 }
 
-                if (options.merged_string !== null && options.merged_string !== undefined) {
-                    const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
-                    const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
-                    const diff = current_merged_archive.replace(options.merged_string, '');
-                    const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-                    fs.appendFileSync(archive_path, diff);
-                }
-
                 let container = null;
 
                 if (file_objs.length > 1) {
@@ -478,28 +488,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
             downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
         }
 
-        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-        if (useYoutubeDLArchive) {
-            const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
-            const archive_path = path.join(archive_folder, `archive_${type}.txt`);
-
-            await fs.ensureDir(archive_folder);
-            await fs.ensureFile(archive_path);
-
-            const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
-            await fs.ensureFile(blacklist_path);
-
-            const merged_path = path.join(archive_folder, `merged_${type}.txt`);
-            await fs.ensureFile(merged_path);
-            // merges blacklist and regular archive
-            let inputPathList = [archive_path, blacklist_path];
-            await mergeFiles(inputPathList, merged_path);
-
-            options.merged_string = await fs.readFile(merged_path, "utf8");
-
-            downloadConfig.push('--download-archive', merged_path);
-        }
-
         if (config_api.getConfigItem('ytdl_include_thumbnail')) {
             downloadConfig.push('--write-thumbnail');
         }
@@ -651,13 +639,3 @@ exports.generateNFOFile = (info, output_path) => {
     const xml = doc.end({ prettyPrint: true });
     fs.writeFileSync(output_path, xml);
 }
-
-function getArchiveFolder(fileFolderPath, options, user_uid) {
-    if (options.customArchivePath) {
-        return path.join(options.customArchivePath);
-    } else if (user_uid) {
-        return path.join(fileFolderPath, 'archives');
-    } else {
-        return path.join('appdata', 'archives');
-    }
-}

backend/subscriptions.js

@@ -3,6 +3,7 @@ const path = require('path');
 const youtubedl = require('youtube-dl');
 
 const config_api = require('./config');
+const archive_api = require('./archive');
 const utils = require('./utils');
 const logger = require('./logger');
 
@@ -160,10 +161,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     let basePath = null;
     basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                         : config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
     const appendedBasePath = getAppendedBasePath(sub, basePath);
     const name = file;
     let retrievedID = null;
+    let retrievedExtractor = null;
 
     await db_api.removeRecord('files', {uid: file_uid});
 
@@ -182,7 +183,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
     ]);
 
     if (jsonExists) {
-        retrievedID = fs.readJSONSync(jsonPath)['id'];
+        const info_json = fs.readJSONSync(jsonPath);
+        retrievedID = info_json['id'];
+        retrievedExtractor = info_json['extractor'];
         await fs.unlink(jsonPath);
     }
 
@@ -200,11 +203,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
             return false;
         } else {
             // check if the user wants the video to be redownloaded (deleteForever === false)
-            if (useArchive && retrievedID) {
+            const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-                const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
+            if (useArchive && !deleteForever) {
-
+                await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
-                // Remove file ID from the archive file, and write it to the blacklist (if enabled)
-                await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
             }
             return true;
         }
@@ -335,8 +336,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     else
         basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
 
-    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-
     let appendedBasePath = getAppendedBasePath(sub, basePath);
 
     const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -372,21 +371,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         downloadConfig.push(...customArgsArray);
     }
 
-    let archive_dir = null;
-    let archive_path = null;
-
-    if (useArchive && !redownload) {
-        if (sub.archive) {
-            archive_dir = sub.archive;
-            if (sub.type && sub.type === 'audio') {
-                archive_path = path.join(archive_dir, 'merged_audio.txt');
-            } else {
-                archive_path = path.join(archive_dir, 'merged_video.txt');
-            }            
-        }
-        downloadConfig.push('--download-archive', archive_path);
-    }
-
     if (sub.timerange && !redownload) {
         downloadConfig.push('--dateafter', sub.timerange);
     }
@@ -429,7 +413,14 @@ async function getFilesToDownload(sub, output_jsons) {
             if (file_with_path_exists) {
                 // or maybe just overwrite???
                 logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+                continue;
             }
+            const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+            if (useYoutubeDLArchive) {
+                const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
+                if (exists_in_archive) continue;
+            }
+
             files_to_download.push(output_json);
         }
     }

backend/test/tests.js

@@ -625,6 +625,7 @@ describe('Tasks', function() {
 
 describe('Archive', async function() {
     beforeEach(async function() {
+        await db_api.connectToDB();
         await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
     });
 
@@ -645,25 +646,50 @@ describe('Archive', async function() {
         const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
         console.log(archive_items);
         assert(archive_items.length === 4);
-        assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor2').length === 1);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
-        assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor1').length === 3);
+        assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
 
         const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
         assert(success);
     });
 
     it('Get archive', async function() {
-        await archive_api.addToArchive('testextractor1', 'video', 'testing1', 'test_user');
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
-        await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user');
 
-        const archive_item1 = await db_api.getRecord('archives', {key: {extractor: 'testextractor1', id: 'testing1'}});
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
-        const archive_item2 = await db_api.getRecord('archives', {key: {extractor: 'testextractor2', id: 'testing1'}});
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
 
         assert(archive_item1 && archive_item2);
+    });
 
-        const count = await db_api.getRecords('archives', {key: {id: 'testing1'}}, true);
+    it('Archive duplicates', async function() {
-        assert(count === 2);
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+
+        await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
+
+        const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
+        assert(count === 3);
+    });
+
+    it('Remove from archive', async function() {
+        await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
+        await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
+
+        const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
+        assert(success);
+
+        const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
+        assert(!!archive_item1);
+        
+        const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
+        assert(!archive_item2);
+
+        const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
+        assert(!!archive_item3);
     });
 });
 

src/api-types/models/DownloadArchiveRequest.ts

@@ -2,8 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { FileType } from './FileType';
+
 export type DownloadArchiveRequest = {
-    sub: {
+    type?: FileType;
-archive_dir: string;
+    sub_id?: string;
-};
 };

src/app/posts.services.ts

@@ -434,8 +434,8 @@ export class PostsService implements CanActivate {
         return this.http.post<SuccessObject>(this.path + 'uploadCookies', fileFormData, this.httpOptions);
     }
 
-    downloadArchive(sub) {
+    downloadArchive(type: FileType, sub_id: string) {
-        const body: DownloadArchiveRequest = {sub: sub};
+        const body: DownloadArchiveRequest = {type: type, sub_id: sub_id};
         return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params});
     }