diff --git a/backend/app.js b/backend/app.js index 44194bd..249224d 100644 --- a/backend/app.js +++ b/backend/app.js @@ -101,7 +101,6 @@ let backendPort = null; let useDefaultDownloadingAgent = null; let customDownloadingAgent = null; let allowSubscriptions = null; -let archivePath = path.join(__dirname, 'appdata', 'archives'); // other needed values let url_domain = null; @@ -506,7 +505,7 @@ async function loadConfig() { db_api.database_initialized_bs.next(true); // creates archive path if missing - await fs.ensureDir(archivePath); + await fs.ensureDir(utils.getArchiveFolder()); // check migrations await checkMigrations(); diff --git a/backend/db.js b/backend/db.js index e35ddca..99d781d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -494,8 +494,7 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => { let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); if (useYoutubeDLArchive) { - const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const archive_path = uuid ? path.join(usersFileFolder, uuid, 'archives', `archive_${type}.txt`) : path.join('appdata', 'archives', `archive_${type}.txt`); + const archive_path = utils.getArchiveFolder(type, uuid); // get ID from JSON @@ -503,14 +502,8 @@ exports.deleteFile = async (uid, uuid = null, blacklistMode = false) => { let id = null; if (jsonobj) id = jsonobj.id; - // use subscriptions API to remove video from the archive file, and write it to the blacklist - if (await fs.pathExists(archive_path)) { - const line = id ? await utils.removeIDFromArchive(archive_path, id) : null; - if (blacklistMode && line) await writeToBlacklist(type, line); - } else { - logger.info('Could not find archive file for audio files. Creating...'); - await fs.close(await fs.open(archive_path, 'w')); - } + // Remove file ID from the archive file, and write it to the blacklist (if enabled) + await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode); } if (jsonExists) await fs.unlink(jsonPath); @@ -1110,15 +1103,3 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => { }); return return_val; } - -// archive helper functions - -async function writeToBlacklist(type, line) { - const archivePath = path.join(__dirname, 'appdata', 'archives'); - let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); - // adds newline to the beginning of the line - line.replace('\n', ''); - line.replace('\r', ''); - line = '\n' + line; - await fs.appendFile(blacklistPath, line); -} diff --git a/backend/downloader.js b/backend/downloader.js index aef46f1..91fcb98 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -18,8 +18,6 @@ const db_api = require('./db'); const mutex = new Mutex(); let should_check_downloads = true; -const archivePath = path.join(__dirname, 'appdata', 'archives'); - if (db_api.database_initialized) { setupDownloads(); } else { @@ -242,6 +240,7 @@ async function downloadQueuedFile(download_uid) { return new Promise(async resolve => { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); + const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path'); await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true}); const url = download['url']; @@ -249,9 +248,11 @@ async function downloadQueuedFile(download_uid) { const options = download['options']; const args = download['args']; const category = download['category']; - let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix + let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; if (options.customFileFolderPath) { fileFolderPath = options.customFileFolderPath; + } else if (download['user_uid']) { + fileFolderPath = path.join(usersFolderPath, download['user_uid'], type); } fs.ensureDirSync(fileFolderPath); @@ -375,13 +376,19 @@ async function downloadQueuedFile(download_uid) { exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); + const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path'); const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; const globalArgs = config_api.getConfigItem('ytdl_custom_args'); const useCookies = config_api.getConfigItem('ytdl_use_cookies'); const is_audio = type === 'audio'; - let fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; + let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix + if (options.customFileFolderPath) { + fileFolderPath = options.customFileFolderPath; + } else if (user_uid) { + fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath); + } if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath; @@ -628,6 +635,6 @@ function getArchiveFolder(fileFolderPath, options, user_uid) { } else if (user_uid) { return path.join(fileFolderPath, 'archives'); } else { - return path.join(archivePath); + return path.join('appdata', 'archives'); } } \ No newline at end of file diff --git a/backend/subscriptions.js b/backend/subscriptions.js index bad2064..8164eb0 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -196,12 +196,11 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, return false; } else { // check if the user wants the video to be redownloaded (deleteForever === false) - if (!deleteForever && useArchive && sub.archive && retrievedID) { - const archive_path = path.join(sub.archive, 'archive.txt') - // if archive exists, remove line with video ID - if (await fs.pathExists(archive_path)) { - utils.removeIDFromArchive(archive_path, retrievedID); - } + if (useArchive && retrievedID) { + const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub); + + // Remove file ID from the archive file, and write it to the blacklist (if enabled) + await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever); } return true; } @@ -322,7 +321,7 @@ function generateOptionsForSubscriptionDownload(sub, user_uid) { selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null, customFileFolderPath: getAppendedBasePath(sub, basePath), customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`, - customArchivePath: path.join(__dirname, basePath, 'archives', sub.name), + customArchivePath: path.join(basePath, 'archives', sub.name), additionalArgs: sub.custom_args } diff --git a/backend/test/tests.js b/backend/test/tests.js index 9236a42..15856c9 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -590,4 +590,32 @@ describe('Tasks', function() { const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'}); assert(dummy_task_obj['data']); }); +}); + +describe('Archive', async function() { + const archive_path = path.join('test', 'archives'); + fs.ensureDirSync(archive_path); + const archive_file_path = path.join(archive_path, 'archive_video.txt'); + const blacklist_file_path = path.join(archive_path, 'blacklist_video.txt'); + beforeEach(async function() { + if (fs.existsSync(archive_file_path)) fs.unlinkSync(archive_file_path); + fs.writeFileSync(archive_file_path, 'youtube testing1\nyoutube testing2\nyoutube testing3\n'); + + if (fs.existsSync(blacklist_file_path)) fs.unlinkSync(blacklist_file_path); + fs.writeFileSync(blacklist_file_path, ''); + }); + + it('Delete from archive', async function() { + await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', false); + const new_archive = fs.readFileSync(archive_file_path); + assert(!new_archive.includes('testing2')); + }); + + it('Delete from archive - blacklist', async function() { + await utils.deleteFileFromArchive('N/A', 'video', archive_path, 'testing2', true); + const new_archive = fs.readFileSync(archive_file_path); + const new_blacklist = fs.readFileSync(blacklist_file_path); + assert(!new_archive.includes('testing2')); + assert(new_blacklist.includes('testing2')); + }); }); \ No newline at end of file diff --git a/backend/utils.js b/backend/utils.js index 1d44681..2ccd00b 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -218,8 +218,11 @@ function deleteJSONFile(file_path, type) { if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); } -async function removeIDFromArchive(archive_path, id) { - let data = await fs.readFile(archive_path, {encoding: 'utf-8'}); +// archive helper functions + +async function removeIDFromArchive(archive_path, type, id) { + const archive_file = path.join(archive_path, `archive_${type}.txt`); + const data = await fs.readFile(archive_file, {encoding: 'utf-8'}); if (!data) { logger.error('Archive could not be found.'); return; @@ -236,12 +239,34 @@ async function removeIDFromArchive(archive_path, id) { } } + if (lastIndex === -1) return null; + const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array // UPDATE FILE WITH NEW DATA const updatedData = dataArray.join('\n'); - await fs.writeFile(archive_path, updatedData); - if (line) return line; + await fs.writeFile(archive_file, updatedData); + if (line) return Array.isArray(line) && line.length === 1 ? line[0] : line; +} + +async function writeToBlacklist(archive_folder, type, line) { + let blacklistPath = path.join(archive_folder, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); + // adds newline to the beginning of the line + line.replace('\n', ''); + line.replace('\r', ''); + line = '\n' + line; + await fs.appendFile(blacklistPath, line); +} + +async function deleteFileFromArchive(uid, type, archive_path, id, blacklistMode) { + const archive_file = path.join(archive_path, `archive_${type}.txt`); + if (await fs.pathExists(archive_path)) { + const line = id ? await removeIDFromArchive(archive_path, type, id) : null; + if (blacklistMode && line) await writeToBlacklist(archive_path, type, line); + } else { + logger.info(`Could not find archive file for file ${uid}. Creating...`); + await fs.close(await fs.open(archive_file, 'w')); + } } function durationStringToNumber(dur_str) { @@ -471,6 +496,25 @@ const searchObjectByString = function(o, s) { return o; } +function getArchiveFolder(type, user_uid = null, sub = null) { + const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path'); + const subsFolderPath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + if (user_uid) { + if (sub) { + return path.join(usersFolderPath, user_uid, 'subscriptions', 'archives', sub.name); + } else { + return path.join(usersFolderPath, user_uid, type, 'archives'); + } + } else { + if (sub) { + return path.join(subsFolderPath, 'archives', sub.name); + } else { + return path.join('appdata', 'archives'); + } + } +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { @@ -500,6 +544,8 @@ module.exports = { fixVideoMetadataPerms: fixVideoMetadataPerms, deleteJSONFile: deleteJSONFile, removeIDFromArchive: removeIDFromArchive, + writeToBlacklist: writeToBlacklist, + deleteFileFromArchive: deleteFileFromArchive, getDownloadedFilesByType: getDownloadedFilesByType, createContainerZipFile: createContainerZipFile, durationStringToNumber: durationStringToNumber, @@ -516,5 +562,6 @@ module.exports = { restartServer: restartServer, injectArgs: injectArgs, searchObjectByString: searchObjectByString, + getArchiveFolder: getArchiveFolder, File: File } diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index b14f9cd..e26e830 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -34,7 +34,7 @@ restoreDelete and redownload