TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-17 18:20:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Implemented basic db-based archive functionality, converted old archive functionality to new system

Browse Source
This commit is contained in:
Tzahi12345
2023-01-23 23:17:58 -05:00
parent 54208ce6ce
commit 840e12db71
9 changed files with 120 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Public API v1.yaml
View File

@@ -1632,6 +1632,9 @@ components:
$ref: '#/components/schemas/FileType'
cropFileSettings:
$ref: '#/components/schemas/CropFileSettings'
ignoreArchive:
type: boolean
description: If using youtube-dl archive, download will ignore it
DownloadResponse:
type: object
properties:
@@ -2080,17 +2083,12 @@ components:
type:
$ref: '#/components/schemas/FileType'
DownloadArchiveRequest:
required:
- sub
type: object
properties:
sub:
required:
- archive_dir
type: object
properties:
archive_dir:
type: string
type:
$ref: '#/components/schemas/FileType'
sub_id:
type: string
UpdaterStatus:
required:
- details

21
backend/app.js
View File

@@ -522,9 +522,6 @@ async function loadConfig() {
db_api.database_initialized = true;
db_api.database_initialized_bs.next(true);
// creates archive path if missing
await fs.ensureDir(utils.getArchiveFolder());
// check migrations
await checkMigrations();
@@ -830,7 +827,8 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword,
ui_uid: req.body.ui_uid,
cropFileSettings: req.body.cropFileSettings
cropFileSettings: req.body.cropFileSettings,
ignoreArchive: req.body.ignoreArchive
};
const download = await downloader_api.createDownload(url, type, options, user_uid);
@@ -1517,15 +1515,18 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
});
app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
let sub = req.body.sub;
let archive_dir = sub.archive;
const uuid = req.isAuthenticated() ? req.user.uid : null;
const sub_id = req.body.sub_id;
const type = req.body.type;
let full_archive_path = path.join(archive_dir, 'archive.txt');
const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
if (await fs.pathExists(full_archive_path)) {
res.sendFile(full_archive_path);
if (archive_text !== null && archive_text !== undefined) {
res.setHeader('Content-type', "application/octet-stream");
res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
res.send(archive_text);
} else {
res.sendStatus(404);
res.sendStatus(400);
}
});

23
backend/archive.js
View File

@@ -3,20 +3,27 @@ const fs = require('fs-extra');
const db_api = require('./db');
exports.generateArchive = async (user_uid = null, sub_id = null) => {
const archive_items = await db_api.getRecords('archives', {user_uid: user_uid, sub_id: sub_id});
exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
const filter = {user_uid: user_uid, sub_id: sub_id};
if (type) filter['type'] = type;
const archive_items = await db_api.getRecords('archives', filter);
const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
return archive_item_lines.join('\n');
}
exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
const success = await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}, type: type});
const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
return success;
}
exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return success;
}
exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
const archive_item = await db_api.getRecord('archives', {'key.extractor': extractor, 'key.id': id, type: type, user_uid: user_uid, sub_id: sub_id});
const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return !!archive_item;
}
@@ -37,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id =
// we can't do a bulk write because we need to avoid duplicate archive items existing in db
const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}});
await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id});
archive_import_count++;
}
return archive_import_count;
@@ -71,10 +78,8 @@ exports.importArchives = async () => {
const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => {
return {
key: {
extractor: extractor,
id: id
},
extractor: extractor,
id: id,
type: type,
user_uid: user_uid ? user_uid : null,
sub_id: sub_id ? sub_id : null

23
backend/db.js
View File

@@ -64,8 +64,7 @@ const tables = {
primary_key: 'uid'
},
archives: {
name: 'archives',
primary_key: 'key'
name: 'archives'
},
test: {
name: 'test'
@@ -510,16 +509,22 @@ exports.deleteFile = async (uid, blacklistMode = false) => {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = utils.getArchiveFolder(type, file_obj.user_uid, file_obj.sub_id ? (await exports.getRecord('subscriptions', {id: file_obj.sub_id})) : null);
// get id/extractor from JSON
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null;
if (jsonobj) id = jsonobj.id;
const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let retrievedID = null;
let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode);
if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
}
if (jsonExists) await fs.unlink(jsonPath);

58
backend/downloader.js
View File

@@ -15,6 +15,7 @@ const categories_api = require('./categories');
const utils = require('./utils');
const db_api = require('./db');
const notifications_api = require('./notifications');
const archive_api = require('./archive');
const mutex = new Mutex();
let should_check_downloads = true;
@@ -202,6 +203,20 @@ async function collectInfo(download_uid) {
return;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
if (exists_in_archive) {
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
logger.warn(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error);
return;
}
}
}
let category = null;
// check if it fits into a category. If so, then get info again using new args
@@ -353,19 +368,14 @@ async function downloadQueuedFile(download_uid) {
// registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj);
}
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null;
if (file_objs.length > 1) {
@@ -478,28 +488,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
@@ -651,13 +639,3 @@ exports.generateNFOFile = (info, output_path) => {
const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml);
}
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join('appdata', 'archives');
}
}

39
backend/subscriptions.js
View File

@@ -3,6 +3,7 @@ const path = require('path');
const youtubedl = require('youtube-dl');
const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils');
const logger = require('./logger');
@@ -160,10 +161,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
let basePath = null;
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file;
let retrievedID = null;
let retrievedExtractor = null;
await db_api.removeRecord('files', {uid: file_uid});
@@ -182,7 +183,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
]);
if (jsonExists) {
retrievedID = fs.readJSONSync(jsonPath)['id'];
const info_json = fs.readJSONSync(jsonPath);
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
await fs.unlink(jsonPath);
}
@@ -200,11 +203,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
return false;
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
if (useArchive && retrievedID) {
const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub);
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
}
return true;
}
@@ -335,8 +336,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = getAppendedBasePath(sub, basePath);
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@@ -372,21 +371,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...customArgsArray);
}
let archive_dir = null;
let archive_path = null;
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
if (sub.type && sub.type === 'audio') {
archive_path = path.join(archive_dir, 'merged_audio.txt');
} else {
archive_path = path.join(archive_dir, 'merged_video.txt');
}
}
downloadConfig.push('--download-archive', archive_path);
}
if (sub.timerange && !redownload) {
downloadConfig.push('--dateafter', sub.timerange);
}
@@ -429,7 +413,14 @@ async function getFilesToDownload(sub, output_jsons) {
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
}
files_to_download.push(output_json);
}
}

44
backend/test/tests.js
View File

@@ -625,6 +625,7 @@ describe('Tasks', function() {
describe('Archive', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
});
@@ -645,25 +646,50 @@ describe('Archive', async function() {
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
console.log(archive_items);
assert(archive_items.length === 4);
assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor1').length === 3);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
assert(success);
});
it('Get archive', async function() {
await archive_api.addToArchive('testextractor1', 'video', 'testing1', 'test_user');
await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user');
await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
const archive_item1 = await db_api.getRecord('archives', {key: {extractor: 'testextractor1', id: 'testing1'}});
const archive_item2 = await db_api.getRecord('archives', {key: {extractor: 'testextractor2', id: 'testing1'}});
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(archive_item1 && archive_item2);
});
const count = await db_api.getRecords('archives', {key: {id: 'testing1'}}, true);
assert(count === 2);
it('Archive duplicates', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
assert(count === 3);
});
it('Remove from archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
assert(success);
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
assert(!!archive_item1);
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(!archive_item2);
const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
assert(!!archive_item3);
});
});

7
src/api-types/models/DownloadArchiveRequest.ts
View File

@@ -2,8 +2,9 @@
/* tslint:disable */
/* eslint-disable */
import type { FileType } from './FileType';
export type DownloadArchiveRequest = {
sub: {
archive_dir: string;
};
type?: FileType;
sub_id?: string;
};

4
src/app/posts.services.ts
View File

@@ -434,8 +434,8 @@ export class PostsService implements CanActivate {
return this.http.post<SuccessObject>(this.path + 'uploadCookies', fileFormData, this.httpOptions);
}
downloadArchive(sub) {
const body: DownloadArchiveRequest = {sub: sub};
downloadArchive(type: FileType, sub_id: string) {
const body: DownloadArchiveRequest = {type: type, sub_id: sub_id};
return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params});
}