TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 12:00:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added ability to reset tasks

Browse Source 
Refactored youtube-dl updating and added youtube-dl update task
This commit is contained in:
Isaac Abadi
2022-04-21 22:04:45 -04:00
parent d2d125743e
commit b53d9c9710
8 changed files with 298 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Public API v1.yaml
View File

@@ -881,10 +881,30 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/GetAllTasksResponse'
/api/resetTasks:
post:
tags:
- tasks
summary: Resets all tasks
operationId: post-api-reset-tasks
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/runTask:
post:
tags:
- tasks
summary: Runs one task
operationId: post-api-run-task
responses:
@@ -901,6 +921,8 @@ paths:
$ref: '#/components/schemas/GetTaskRequest'
/api/confirmTask:
post:
tags:
- tasks
summary: Confirms a task
operationId: post-api-confirm-task
responses:
@@ -917,6 +939,8 @@ paths:
$ref: '#/components/schemas/GetTaskRequest'
/api/cancelTask:
post:
tags:
- tasks
summary: Cancels a task
operationId: post-api-cancel-task
responses:
@@ -933,6 +957,8 @@ paths:
$ref: '#/components/schemas/GetTaskRequest'
/api/updateTaskSchedule:
post:
tags:
- tasks
summary: Updates task schedule
operationId: post-api-update-task-schedule
responses:
@@ -949,6 +975,8 @@ paths:
$ref: '#/components/schemas/UpdateTaskScheduleRequest'
/api/updateTaskData:
post:
tags:
- tasks
summary: Updates task data
operationId: post-api-update-task-data
responses:
@@ -965,6 +993,8 @@ paths:
$ref: '#/components/schemas/UpdateTaskDataRequest'
/api/getDBBackups:
post:
tags:
- tasks
summary: Get database backups
operationId: post-api-get-database-backups
responses:
@@ -981,6 +1011,8 @@ paths:
type: object
/api/restoreDBBackup:
post:
tags:
- tasks
summary: Restore database backup
operationId: post-api-restore-database-backup
responses:

197
backend/app.js
View File

@@ -13,7 +13,6 @@ const unzipper = require('unzipper');
const db_api = require('./db');
const utils = require('./utils')
const low = require('lowdb')
const ProgressBar = require('progress');
const fetch = require('node-fetch');
const URL = require('url').URL;
const CONSTS = require('./consts')
@@ -32,8 +31,7 @@ const tasks_api = require('./tasks');
const subscriptions_api = require('./subscriptions');
const categories_api = require('./categories');
const twitch_api = require('./twitch');
const is_windows = process.platform === 'win32';
const youtubedl_api = require('./youtube-dl');
var app = express();
@@ -357,34 +355,6 @@ async function downloadReleaseFiles(tag) {
});
}
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
len = parseInt(res.headers.get("Content-Length"), 10);
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();
});
});
}
async function downloadReleaseZip(tag) {
return new Promise(async resolve => {
// get name of zip file, which depends on the version
@@ -395,7 +365,7 @@ async function downloadReleaseZip(tag) {
let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`);
// download zip from release
await fetchFile(latest_zip_link, output_path, 'update ' + tag);
await utils.fetchFile(latest_zip_link, output_path, 'update ' + tag);
resolve(true);
});
@@ -708,156 +678,8 @@ async function getUrlInfos(url) {
async function startYoutubeDL() {
// auto update youtube-dl
await autoUpdateYoutubeDL();
}
// auto updates the underlying youtube-dl binary, not YoutubeDL-Material
async function autoUpdateYoutubeDL() {
const download_sources = {
'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
'func': downloadLatestYoutubeDLBinary
},
'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
'func': downloadLatestYoutubeDLCBinary
},
'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
'func': downloadLatestYoutubeDLPBinary
}
}
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const tags_url = download_sources[default_downloader]['tags_url'];
// get current version
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
if (!current_app_details_exists) {
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
}
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
let current_version = current_app_details['version'];
let current_downloader = current_app_details['downloader'];
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(false);
return;
}
}
// got version, now let's check the latest version from the youtube-dl API
fetch(tags_url, {method: 'Get'})
.then(async res => res.json())
.then(async (json) => {
// check if the versions are different
if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`)
resolve(false);
return false;
}
const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
// versions different or different downloader is being used, download new update
logger.info(`Found new update for ${default_downloader}. Updating binary...`);
try {
await checkExistsWithTimeout(stored_binary_path, 10000);
} catch(e) {
logger.error(`Failed to update ${default_downloader} - ${e}`);
}
await download_sources[default_downloader]['func'](latest_update_version);
resolve(true);
} else {
resolve(false);
}
})
.catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`)
logger.error(err);
resolve(false);
return false;
});
});
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version;
details_json['downloader'] = downloader;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
}
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
if (watcher) watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
});
const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
}
app.use(function(req, res, next) {
@@ -1886,6 +1708,17 @@ app.post('/api/getTasks', optionalJwt, async (req, res) => {
res.send({tasks: tasks});
});
app.post('/api/resetTasks', optionalJwt, async (req, res) => {
const tasks_keys = Object.keys(tasks_api.TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
tasks_api.TASKS[task_key]['job'] = null;
}
await db_api.removeAllRecords('tasks');
await tasks_api.setupTasks();
res.send({success: true});
});
app.post('/api/getTask', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const task = await db_api.getRecord('tasks', {key: task_key});

23
backend/tasks.js
View File

@@ -1,5 +1,5 @@
const utils = require('./utils');
const db_api = require('./db');
const youtubedl_api = require('./youtube-dl');
const fs = require('fs-extra');
const logger = require('./logger');
@@ -27,6 +27,12 @@ const TASKS = {
confirm: removeDuplicates,
title: 'Find duplicate files in DB',
job: null
},
youtubedl_update_check: {
run: youtubedl_api.checkForYoutubeDLUpdate,
confirm: youtubedl_api.updateYoutubeDL,
title: 'Update youtube-dl',
job: null
}
}
@@ -52,20 +58,22 @@ function scheduleJob(task_key, schedule) {
return;
}
// remove schedule if it's a one-time task
if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
// we're just "running" the task, any confirmation should be user-initiated
exports.executeRun(task_key);
});
}
if (db_api.database_initialized) {
setupTasks();
exports.setupTasks();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupTasks();
if (init) exports.setupTasks();
});
}
const setupTasks = async () => {
exports.setupTasks = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
@@ -90,6 +98,11 @@ const setupTasks = async () => {
// schedule task and save job
if (task_in_db['schedule']) {
// prevent timestamp schedules from being set to the past
if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) {
await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
continue;
}
TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
}
}
@@ -113,7 +126,7 @@ exports.executeRun = async (task_key) => {
// don't set running to true when backup up DB as it will be stick "running" if restored
if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
const data = await TASKS[task_key].run();
await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false});
await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
logger.verbose(`Finished running task ${task_key}`);
}

69
backend/utils.js
View File

@@ -1,10 +1,13 @@
const fs = require('fs-extra')
const path = require('path')
const fs = require('fs-extra');
const path = require('path');
const ffmpeg = require('fluent-ffmpeg');
const archiver = require('archiver');
const fetch = require('node-fetch');
const ProgressBar = require('progress');
const config_api = require('./config');
const logger = require('./logger');
const CONSTS = require('./consts')
const archiver = require('archiver');
const CONSTS = require('./consts');
const is_windows = process.platform === 'win32';
@@ -356,6 +359,62 @@ async function cropFile(file_path, start, end, ext) {
});
}
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
if (watcher) watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
});
}
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
len = parseInt(res.headers.get("Content-Length"), 10);
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();
});
});
}
// objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -397,5 +456,7 @@ module.exports = {
cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
checkExistsWithTimeout: checkExistsWithTimeout,
fetchFile: fetchFile,
File: File
}

127
backend/youtube-dl.js Normal file
View File

@@ -0,0 +1,127 @@
const fs = require('fs-extra');
const fetch = require('node-fetch');
const logger = require('./logger');
const utils = require('./utils');
const CONSTS = require('./consts');
const config_api = require('./config.js');
const is_windows = process.platform === 'win32';
const download_sources = {
'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
'func': downloadLatestYoutubeDLBinary
},
'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
'func': downloadLatestYoutubeDLCBinary
},
'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
'func': downloadLatestYoutubeDLPBinary
}
}
exports.checkForYoutubeDLUpdate = async () => {
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const tags_url = download_sources[default_downloader]['tags_url'];
// get current version
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
if (!current_app_details_exists) {
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
}
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
let current_version = current_app_details['version'];
let current_downloader = current_app_details['downloader'];
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(null);
return;
}
}
// got version, now let's check the latest version from the youtube-dl API
fetch(tags_url, {method: 'Get'})
.then(async res => res.json())
.then(async (json) => {
// check if the versions are different
if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`)
resolve(null);
return;
}
const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
// versions different or different downloader is being used, download new update
resolve(latest_update_version);
} else {
resolve(null);
}
return;
})
.catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`)
logger.error(err);
resolve(null);
return;
});
});
}
exports.updateYoutubeDL = async (latest_update_version) => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await download_sources[default_downloader]['func'](latest_update_version);
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version;
details_json['downloader'] = downloader;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
}

14
src/app/components/tasks/tasks.component.html
View File

@@ -50,11 +50,18 @@
<mat-cell *matCellDef="let element">
<div class="container">
<div class="row justify-content-center">
<div *ngIf="element.data?.uids?.length > 0" class="col-12 mt-2">
<div *ngIf="element.data?.uids?.length > 0 || (!element.data?.uids && element.data)" class="col-12 mt-2" style="display: flex; justify-content: center;">
<ng-container>
<button (click)="confirmTask(element.key)" [disabled]="element.running || element.confirming" mat-stroked-button>
<ng-container *ngIf="element.key == 'missing_files_check'" i18n="Clear missing files from DB">Clear missing files from DB:</ng-container>
<ng-container *ngIf="element.key == 'duplicate_files_check'" i18n="Clear duplicate files from DB">Clear duplicate files from DB:</ng-container>&nbsp;{{element.data.uids.length}}
<ng-container *ngIf="element.key == 'missing_files_check'">
<ng-container i18n="Clear missing files from DB">Clear missing files from DB:</ng-container>{{element.data.uids.length}}
</ng-container>
<ng-container *ngIf="element.key == 'duplicate_files_check'">
<ng-container i18n="Clear duplicate files from DB">Clear duplicate files from DB:</ng-container>&nbsp;{{element.data.uids.length}}
</ng-container>
<ng-container *ngIf="element.key == 'youtubedl_update_check'">
<ng-container i18n="Update binary to">Update binary to:</ng-container>&nbsp;{{element.data}}
</ng-container>
</button>
</ng-container>
</div>
@@ -80,6 +87,7 @@
</div>
<button style="margin-top: 10px; margin-left: 5px;" mat-stroked-button (click)="openRestoreDBBackupDialog()" i18n="Restore DB from backup button">Restore DB from backup</button>
<button style="margin-top: 10px; margin-left: 5px;" mat-stroked-button (click)="resetTasks()" color="warn" i18n="Reset tasks button">Reset tasks</button>
</div>
<div *ngIf="(!tasks || tasks.length === 0) && tasks_retrieved">

26
src/app/components/tasks/tasks.component.ts
View File

@@ -3,6 +3,7 @@ import { MatDialog } from '@angular/material/dialog';
import { MatPaginator } from '@angular/material/paginator';
import { MatSort } from '@angular/material/sort';
import { MatTableDataSource } from '@angular/material/table';
import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component';
import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
import { PostsService } from 'app/posts.services';
@@ -125,6 +126,31 @@ export class TasksComponent implements OnInit {
})
}
resetTasks(): void {
const dialogRef = this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: $localize`Reset tasks`,
dialogText: $localize`Would you like to reset your tasks? All your schedules will be removed as well.`,
submitText: $localize`Reset`,
warnSubmitColor: true
}
});
dialogRef.afterClosed().subscribe(confirmed => {
if (confirmed) {
this.postsService.resetTasks().subscribe(res => {
if (res['success']) {
this.postsService.openSnackBar($localize`Tasks successfully reset!`);
} else {
this.postsService.openSnackBar($localize`Failed to reset tasks!`);
}
}, err => {
this.postsService.openSnackBar($localize`Failed to reset tasks!`);
console.error(err);
});
}
});
}
}
export interface Task {

4
src/app/posts.services.ts
View File

@@ -591,6 +591,10 @@ export class PostsService implements CanActivate {
return this.http.post<SuccessObject>(this.path + 'getTasks', {}, this.httpOptions);
}
resetTasks() {
return this.http.post<SuccessObject>(this.path + 'resetTasks', {}, this.httpOptions);
}
getTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<GetTaskResponse>(this.path + 'getTask', body, this.httpOptions);