TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added additional protections to verify that the DB is initialized before downloader does

Browse Source 
Began work on watching entire subscriptions as a playlist

Subscriptions now use the new download manager to download files
This commit is contained in:
Isaac Abadi
2021-08-21 21:54:40 -06:00
parent f7b152fcf6
commit 9f5b6122fa
11 changed files with 151 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/app.js
View File

@@ -61,7 +61,7 @@ config_api.initialize();
db_api.initialize(db, users_db);
auth_api.initialize(db_api);
downloader_api.initialize(db_api);
subscriptions_api.initialize(db_api);
subscriptions_api.initialize(db_api, downloader_api);
categories_api.initialize(db_api);
// Set some defaults
@@ -533,6 +533,8 @@ async function loadConfig() {
// connect to DB
await db_api.connectToDB();
db_api.database_initialized = true;
db_api.database_initialized_bs.next(true);
// creates archive path if missing
await fs.ensureDir(archivePath);

3
backend/db.js
View File

@@ -9,10 +9,13 @@ const logger = require('./logger');
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync');
const { BehaviorSubject } = require('rxjs');
const local_adapter = new FileSync('./appdata/local_db.json');
const local_db = low(local_adapter);
let database = null;
exports.database_initialized = false;
exports.database_initialized_bs = new BehaviorSubject(false);
const tables = {
files: {

33
backend/downloader.js
View File

@@ -26,16 +26,24 @@ function setDB(input_db_api) { db_api = input_db_api }
exports.initialize = (input_db_api) => {
setDB(input_db_api);
categories_api.initialize(db_api);
setupDownloads();
if (db_api.database_initialized) {
setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
});
}
}
exports.createDownload = async (url, type, options, user_uid = null) => {
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
return await mutex.runExclusive(async () => {
const download = {
url: url,
type: type,
title: '',
user_uid: user_uid,
sub_id: sub_id,
sub_name: sub_name,
options: options,
uid: uuid(),
step_index: 0,
@@ -116,7 +124,7 @@ async function setupDownloads() {
async function fixDownloadState() {
const downloads = await db_api.getRecords('download_queue');
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
const running_downloads = downloads.filter(download => !download['finished_step']);
const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
for (let i = 0; i < running_downloads.length; i++) {
const running_download = running_downloads[i];
const update_obj = {finished_step: true, paused: true};
@@ -143,7 +151,7 @@ async function checkDownloads() {
return;
});
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step']);
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
for (let i = 0; i < waiting_downloads.length; i++) {
const running_download = waiting_downloads[i];
if (i === 5/*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/) break;
@@ -322,7 +330,7 @@ async function downloadQueuedFile(download_uid) {
}
// registers file in DB
const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, null, options.cropFileSettings);
const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj);
}
@@ -437,13 +445,20 @@ async function generateArgs(url, type, options, user_uid = null) {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = user_uid ? path.join(fileFolderPath, 'archives') : archivePath;
let archive_folder = null;
if (options.customArchivePath) {
archive_folder = path.join(options.customArchivePath);
} else if (user_uid) {
archive_folder = path.join(fileFolderPath, 'archives');
} else {
archive_folder = path.join(archivePath);
}
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
let blacklist_path = user_uid ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
let blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
@@ -471,6 +486,10 @@ async function generateArgs(url, type, options, user_uid = null) {
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
}
if (options.additionalArgs && options.additionalArgs !== '') {
downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);

120
backend/subscriptions.js
View File

@@ -10,11 +10,13 @@ const logger = require('./logger');
const debugMode = process.env.YTDL_MODE === 'debug';
let db_api = null;
let downloader_api = null;
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db_api) {
function initialize(input_db_api, input_downloader_api) {
setDB(input_db_api);
downloader_api = input_downloader_api;
}
async function subscribe(sub, user_uid = null) {
@@ -107,22 +109,6 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
}
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useArchive && !sub.archive) {
// must create the archive
const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
const archive_path = path.join(archive_dir, 'archive.txt');
// creates archive directory and text file if it doesn't exist
fs.ensureDirSync(archive_dir);
fs.ensureFileSync(archive_path);
// updates subscription
sub.archive = archive_dir;
await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
}
// TODO: get even more info
resolve(true);
@@ -256,30 +242,15 @@ async function getVideosForSub(sub, user_uid = null) {
let appendedBasePath = getAppendedBasePath(sub, basePath);
fs.ensureDirSync(appendedBasePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
// get videos
logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
return new Promise(async resolve => {
const preimported_file_paths = [];
const PREIMPORT_INTERVAL = 5000;
const preregister_check = setInterval(async () => {
if (sub.streamingOnly) return;
await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
}, PREIMPORT_INTERVAL);
youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
// cleanup
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
@@ -287,19 +258,21 @@ async function getVideosForSub(sub, user_uid = null) {
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
await handleOutputJSON(sub, output, i === 0, multiUserMode)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
fs.appendFileSync(archive_path, output['id']);
}
}
}
// TODO: reimplement
// const outputs = err.stdout.split(/\r\n|\r|\n/);
// for (let i = 0; i < outputs.length; i++) {
// const output = JSON.parse(outputs[i]);
// await handleOutputJSON(sub, output, i === 0, multiUserMode)
// if (err.stderr.includes(output['id']) && archive_path) {
// // we found a video that errored! add it to the archive to prevent future errors
// if (sub.archive) {
// archive_dir = sub.archive;
// archive_path = path.join(archive_dir, 'archive.txt')
// fs.appendFileSync(archive_path, output['id']);
// }
// }
// }
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
@@ -312,21 +285,30 @@ async function getVideosForSub(sub, user_uid = null) {
resolve(true);
return;
}
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
output_jsons.push(output_json);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
const reset_videos = i === 0;
await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
}
const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
}
resolve(files_to_download);
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid);
@@ -338,10 +320,28 @@ async function getVideosForSub(sub, user_uid = null) {
}, err => {
logger.error(err);
updateSubscriptionProperty(sub, {downloading: false}, user_uid);
clearInterval(preregister_check);
});
}
function generateOptionsForSubscriptionDownload(sub, user_uid) {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
const base_download_options = {
selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
customFileFolderPath: getAppendedBasePath(sub, basePath),
customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
customArchivePath: path.join(__dirname, basePath, 'archives', sub.name)
}
return base_download_options;
}
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
// get basePath
let basePath = null;
@@ -363,7 +363,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
@@ -378,7 +378,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...qualityPath)
if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,');
const customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
@@ -466,6 +466,24 @@ async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_vi
}
}
async function getFilesToDownload(sub, output_jsons) {
const files_to_download = [];
for (let i = 0; i < output_jsons.length; i++) {
const output_json = output_jsons[i];
const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']}));
if (file_missing) {
const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
if (file_with_path_exists) {
// or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
}
files_to_download.push(output_json);
}
}
return files_to_download;
}
async function getSubscriptions(user_uid = null) {
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}

25
src/app/components/downloads/downloads.component.html
View File

@@ -17,6 +17,19 @@
</span>
</mat-cell>
</ng-container>
<!-- Subscription Column -->
<ng-container matColumnDef="subscription">
<mat-header-cell *matHeaderCellDef> <ng-container i18n="Subscription">Subscription</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<ng-container *ngIf="element.sub_name">
{{element.sub_name}}
</ng-container>
<ng-container *ngIf="!element.sub_name">
N/A
</ng-container>
</mat-cell>
</ng-container>
<!-- Stage Column -->
<ng-container matColumnDef="stage">
@@ -41,15 +54,17 @@
<ng-container matColumnDef="actions">
<mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<div *ngIf="!element.finished">
<div>
<ng-container *ngIf="!element.finished">
<button (click)="pauseDownload(element.uid)" *ngIf="!element.paused || !element.finished_step" [disabled]="element.paused && !element.finished_step" mat-icon-button matTooltip="Pause" i18n-matTooltip="Pause"><mat-spinner [diameter]="28" *ngIf="element.paused && !element.finished_step" class="icon-button-spinner"></mat-spinner><mat-icon>pause</mat-icon></button>
<button (click)="resumeDownload(element.uid)" *ngIf="element.paused && element.finished_step" mat-icon-button matTooltip="Resume" i18n-matTooltip="Resume"><mat-icon>play_arrow</mat-icon></button>
<button (click)="cancelDownload(element.uid)" mat-icon-button matTooltip="Cancel" i18n-matTooltip="Cancel"><mat-icon>cancel</mat-icon></button>
</div>
<div *ngIf="element.finished">
<button *ngIf="!element.paused" (click)="cancelDownload(element.uid)" mat-icon-button matTooltip="Cancel" i18n-matTooltip="Cancel"><mat-icon>cancel</mat-icon></button>
</ng-container>
<ng-container *ngIf="element.finished">
<button (click)="watchContent(element)" mat-icon-button matTooltip="Watch content" i18n-matTooltip="Watch content"><mat-icon>smart_display</mat-icon></button>
<button (click)="restartDownload(element.uid)" mat-icon-button matTooltip="Restart" i18n-matTooltip="Restart"><mat-icon>restart_alt</mat-icon></button>
<button (click)="clearDownload(element.uid)" mat-icon-button matTooltip="Clear" i18n-matTooltip="Clear"><mat-icon>delete</mat-icon></button>
</ng-container>
<button *ngIf="element.finished || element.paused" (click)="clearDownload(element.uid)" mat-icon-button matTooltip="Clear" i18n-matTooltip="Clear"><mat-icon>delete</mat-icon></button>
</div>
</mat-cell>
</ng-container>

2
src/app/components/downloads/downloads.component.ts
View File

@@ -52,7 +52,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
3: 'Complete'
}
displayedColumns: string[] = ['date', 'title', 'stage', 'progress', 'actions'];
displayedColumns: string[] = ['date', 'title', 'stage', 'subscription', 'progress', 'actions'];
dataSource = null; // new MatTableDataSource<Download>();
downloads_retrieved = false;

12
src/app/main/main.component.ts
View File

@@ -246,11 +246,6 @@ export class MainComponent implements OnInit {
this.useDefaultDownloadingAgent = this.postsService.config['Advanced']['use_default_downloading_agent'];
this.customDownloadingAgent = this.postsService.config['Advanced']['custom_downloading_agent'];
if (this.youtubeSearchEnabled && this.youtubeAPIKey) {
this.youtubeSearch.initializeAPI(this.youtubeAPIKey);
this.attachToInput();
}
// set final cache items
localStorage.setItem('cached_filemanager_enabled', this.fileManagerEnabled.toString());
@@ -330,6 +325,13 @@ export class MainComponent implements OnInit {
this.setCols();
}
ngAfterViewInit() {
if (this.youtubeSearchEnabled && this.youtubeAPIKey) {
this.youtubeSearch.initializeAPI(this.youtubeAPIKey);
this.attachToInput();
}
}
public setCols() {
if (window.innerWidth <= 350) {
this.files_cols = 1;

23
src/app/player/player.component.ts
View File

@@ -166,18 +166,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
const subscription = res['subscription'];
this.subscription = subscription;
this.type === this.subscription.type;
subscription.videos.forEach(video => {
if (video['uid'] === this.uid) {
this.db_file = video;
this.postsService.incrementViewCount(this.db_file['uid'], this.sub_id, this.uuid).subscribe(res => {}, err => {
console.error('Failed to increment view count');
console.error(err);
});
this.uids = [this.db_file['uid']];
this.show_player = true;
this.parseFileNames();
}
});
this.uids = this.subscription.videos.map(video => video['uid']);
this.parseFileNames();
}, err => {
this.openSnackBar(`Failed to find subscription ${this.sub_id}`, 'Dismiss');
});
@@ -205,7 +195,14 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
for (let i = 0; i < this.uids.length; i++) {
const uid = this.uids[i];
const file_obj = this.playlist_id ? this.db_playlist['file_objs'][i] : this.db_file;
let file_obj = null;
if (this.playlist_id) {
file_obj = this.db_playlist['file_objs'][i];
} else if (this.sub_id) {
file_obj = this.subscription['videos'][i];
} else {
file_obj = this.db_file;
}
const mime_type = file_obj.isAudio ? 'audio/mp3' : 'video/mp4'

1
src/app/subscription/subscription/subscription.component.html
View File

@@ -44,5 +44,6 @@
</div>
</div>
<button class="edit-button" color="primary" (click)="editSubscription()" [disabled]="downloading" mat-fab><mat-icon class="save-icon">edit</mat-icon></button>
<button class="watch-button" color="primary" (click)="watchSubscription()" mat-fab><mat-icon class="save-icon">video_library</mat-icon></button>
<button class="save-button" color="primary" (click)="downloadContent()" [disabled]="downloading" mat-fab><mat-icon class="save-icon">save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="50"></mat-spinner></button>
</div>

6
src/app/subscription/subscription/subscription.component.scss
View File

@@ -67,4 +67,10 @@
.save-icon {
bottom: 1px;
position: relative;
}
.watch-button {
left: 90px;
position: fixed;
bottom: 25px;
}

7
src/app/subscription/subscription/subscription.component.ts
View File

@@ -109,8 +109,7 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
if (this.subscription.streamingOnly) {
this.router.navigate(['/player', {uid: uid, url: url}]);
} else {
this.router.navigate(['/player', {uid: uid,
sub_id: this.subscription.id}]);
this.router.navigate(['/player', {uid: uid}]);
}
}
@@ -171,4 +170,8 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
});
}
watchSubscription() {
this.router.navigate(['/player', {sub_id: this.subscription.id}])
}
}