TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-10 06:41:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #206 from Tzahi12345/downloader-improvements

Browse Source 
Downloader improvements - updated system and bug fixes
This commit is contained in:
Tzahi12345
2020-08-31 15:17:40 -04:00
committed by GitHub
parent 70159813e5 71633950b2
commit 5eccaa13e5
13 changed files with 216 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
backend/app.js
View File

@@ -7,7 +7,7 @@ var path = require('path');
var youtubedl = require('youtube-dl');
var ffmpeg = require('fluent-ffmpeg');
var compression = require('compression');
var https = require('https');
var glob = require("glob")
var multer = require('multer');
var express = require("express");
var bodyParser = require("body-parser");
@@ -1146,12 +1146,29 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now()
timestamp_start: Date.now(),
filesize: null
};
const download = downloads[session][download_uid];
updateDownloads();
// get video info prior to download
const info = await getVideoInfoByURL(url, downloadConfig, download);
if (!info) {
resolve(false);
return;
} else {
// store info in download for future use
download['_filename'] = info['_filename'];
download['filesize'] = utils.getExpectedFileSize(info);
}
const download_checker = setInterval(() => checkDownloadPercent(download), 1000);
// download file
youtubedl.exec(url, downloadConfig, {}, function(err, output) {
clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
download['downloading'] = false;
download['timestamp_end'] = Date.now();
var file_uid = null;
@@ -1164,7 +1181,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
download['error'] = err.stderr;
updateDownloads();
resolve(false);
throw err;
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
download['error'] = 'No output. Check if video already exists in your archive.';
@@ -1407,7 +1424,7 @@ async function generateArgs(url, type, options) {
var youtubePassword = options.youtubePassword;
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'best[ext=mp4]'];
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
@@ -1485,6 +1502,10 @@ async function generateArgs(url, type, options) {
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
@@ -1497,11 +1518,36 @@ async function generateArgs(url, type, options) {
}
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
// downloadConfig.map((arg) => `"${arg}"`);
resolve(downloadConfig);
});
}
async function getVideoInfoByURL(url, args = [], download = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
// actually get info
youtubedl.getInfo(url, new_args, (err, output) => {
if (output) {
resolve(output);
} else {
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
if (download) {
download['error'] = `Failed pre-check for video info: ${err}`;
updateDownloads();
}
resolve(null);
}
});
});
}
// currently only works for single urls
async function getUrlInfos(urls) {
let startDate = Date.now();
@@ -1559,47 +1605,33 @@ function updateDownloads() {
db.assign({downloads: downloads}).write();
}
/*
function checkDownloads() {
for (let [session_id, session_downloads] of Object.entries(downloads)) {
for (let [download_uid, download_obj] of Object.entries(session_downloads)) {
if (download_obj && !download_obj['complete'] && !download_obj['error']
&& download_obj.timestamp_start > timestamp_server_start) {
// download is still running (presumably)
download_obj.percent_complete = getDownloadPercent(download_obj);
}
}
}
}
*/
function checkDownloadPercent(download) {
/*
This is more of an art than a science, we're just selecting files that start with the file name,
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
function getDownloadPercent(download_obj) {
if (!download_obj.final_size) {
if (fs.existsSync(download_obj.expected_json_path)) {
const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8'));
let calculated_filesize = null;
if (file_json['format_id']) {
calculated_filesize = 0;
const formats_used = file_json['format_id'].split('+');
for (let i = 0; i < file_json['formats'].length; i++) {
if (formats_used.includes(file_json['formats'][i]['format_id'])) {
calculated_filesize += file_json['formats'][i]['filesize'];
}
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
be divided by the "total expected bytes."
*/
const file_id = download['file_id'];
const filename = path.format(path.parse(download['_filename'].substring(0, download['_filename'].length-4)));
const resulting_file_size = download['filesize'];
glob(`${filename}*`, (err, files) => {
let sum_size = 0;
files.forEach(file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
download_obj.final_size = calculated_filesize;
} else {
console.log('could not find json file');
}
}
if (fs.existsSync(download_obj.expected_path)) {
const stats = fs.statSync(download_obj.expected_path);
const size = stats.size;
return (size / download_obj.final_size)*100;
} else {
console.log('could not find file');
return 0;
}
});
download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
updateDownloads();
});
}
// youtube-dl functions
@@ -1821,7 +1853,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
const is_playlist = url.includes('playlist');
let result_obj = null;
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
@@ -1833,6 +1865,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
});
app.post('/api/tomp4', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos
var url = req.body.url;
var options = {
customArgs: req.body.customArgs,
@@ -1850,7 +1883,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
const is_playlist = url.includes('playlist');
let result_obj = null;
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
@@ -1878,6 +1911,14 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
playlists = auth_api.getUserPlaylists(req.user.uid, 'audio');
}
mp3s = JSON.parse(JSON.stringify(mp3s));
// add thumbnails if present
mp3s.forEach(mp3 => {
if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath']))
mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']);
});
res.send({
mp3s: mp3s,
playlists: playlists
@@ -1897,6 +1938,14 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) {
playlists = auth_api.getUserPlaylists(req.user.uid, 'video');
}
mp4s = JSON.parse(JSON.stringify(mp4s));
// add thumbnails if present
mp4s.forEach(mp4 => {
if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath']))
mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']);
});
res.send({
mp4s: mp4s,
playlists: playlists
@@ -1981,6 +2030,14 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) {
files = files.concat(sub.videos);
}
files = JSON.parse(JSON.stringify(files));
// add thumbnails if present
files.forEach(file => {
if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath']))
file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']);
});
res.send({
files: files,
playlists: playlists

4
backend/appdata/default.json
View File

@@ -9,7 +9,9 @@
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",

4
backend/config.js
View File

@@ -186,7 +186,9 @@ DEFAULT_CONFIG = {
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",

8
backend/consts.js
View File

@@ -30,6 +30,14 @@ let CONFIG_ITEMS = {
'key': 'ytdl_safe_download_override',
'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
},
'ytdl_include_thumbnail': {
'key': 'ytdl_include_thumbnail',
'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
},
'ytdl_include_metadata': {
'key': 'ytdl_include_metadata',
'path': 'YoutubeDLMaterial.Downloader.include_metadata'
},
// Extra
'ytdl_title_top': {

15
backend/db.js
View File

@@ -26,11 +26,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path);
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path);
if (!sub) {
if (multiUserMode) {
@@ -48,7 +45,13 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
}
}
const file_uid = registerFileDBManual(db_path, file_object)
const file_uid = registerFileDBManual(db_path, file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
}
return file_uid;
}

1
backend/package.json
View File

@@ -37,6 +37,7 @@
"express": "^4.17.1",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
"jsonwebtoken": "^8.5.1",
"lowdb": "^1.0.0",
"md5": "^2.2.1",

4
backend/subscriptions.js
View File

@@ -345,6 +345,10 @@ async function getVideosForSub(sub, user_uid = null) {
}
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {

51
backend/utils.js
View File

@@ -88,6 +88,41 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getExpectedFileSize(info_json) {
if (info_json['filesize']) {
return info_json['filesize'];
}
const formats = info_json['format_id'].split('+');
let expected_filesize = 0;
formats.forEach(format_id => {
info_json.formats.forEach(available_format => {
if (available_format.format_id === format_id && available_format.filesize) {
expected_filesize += available_format.filesize;
}
});
});
return expected_filesize;
}
function fixVideoMetadataPerms(name, type, customPath = null) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
@@ -110,6 +145,19 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
}
}
function deleteJSONFile(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
let json_path = path.join(customPath, name + '.info.json');
let alternate_json_path = path.join(customPath, name + ext + '.info.json');
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
function recFindByExt(base,ext,files,result)
{
files = files || fs.readdirSync(base)
@@ -153,7 +201,10 @@ module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
deleteJSONFile: deleteJSONFile,
getDownloadedFilesByType: getDownloadedFilesByType,
recFindByExt: recFindByExt,
File: File

2
src/app/components/unified-file-card/unified-file-card.component.html
View File

@@ -29,7 +29,7 @@
<div style="padding:5px">
<div *ngIf="!loading && file_obj.thumbnailURL" class="img-div">
<div style="position: relative">
<img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailURL" alt="Thumbnail">
<img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailBlob ? thumbnailBlobURL : file_obj.thumbnailURL" alt="Thumbnail">
<div class="duration-time">
{{file_length}}
</div>

27
src/app/components/unified-file-card/unified-file-card.component.ts
View File

@@ -1,6 +1,7 @@
import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core';
import { MatDialog } from '@angular/material/dialog';
import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component';
import { DomSanitizer } from '@angular/platform-browser';
@Component({
selector: 'app-unified-file-card',
@@ -16,6 +17,10 @@ export class UnifiedFileCardComponent implements OnInit {
type = null;
elevated = false;
// optional vars
thumbnailBlobURL = null;
// input/output
@Input() loading = true;
@Input() theme = null;
@Input() file_obj = null;
@@ -35,12 +40,19 @@ export class UnifiedFileCardComponent implements OnInit {
big: 250x200
*/
constructor(private dialog: MatDialog) { }
constructor(private dialog: MatDialog, private sanitizer: DomSanitizer) { }
ngOnInit(): void {
if (!this.loading) {
this.file_length = fancyTimeFormat(this.file_obj.duration);
}
if (this.file_obj && this.file_obj.thumbnailBlob) {
const mime = getMimeByFilename(this.file_obj.thumbnailPath);
const blob = new Blob([new Uint8Array(this.file_obj.thumbnailBlob.data)], {type: mime});
const bloburl = URL.createObjectURL(blob);
this.thumbnailBlobURL = this.sanitizer.bypassSecurityTrustUrl(bloburl);
}
}
emitDeleteFile(blacklistMode = false) {
@@ -97,3 +109,16 @@ function fancyTimeFormat(time) {
ret += '' + secs;
return ret;
}
function getMimeByFilename(name) {
switch (name.substring(name.length-4, name.length)) {
case '.jpg':
return 'image/jpeg';
case '.png':
return 'image/png';
case 'webp':
return 'image/webp';
default:
return null;
}
}

2
src/app/main/main.component.html
View File

@@ -164,7 +164,7 @@
<br/>
<div class="centered big" id="bar_div" *ngIf="current_download && current_download.downloading; else nofile">
<div class="margined">
<div [ngClass]="(+percentDownloaded > 99)?'make-room-for-spinner':'equal-sizes'" style="display: inline-block; width: 100%; padding-left: 20px" *ngIf="current_download.percent_complete && current_download.percent_complete > 15;else indeterminateprogress">
<div [ngClass]="(+percentDownloaded > 99)?'make-room-for-spinner':'equal-sizes'" style="display: inline-block; width: 100%; padding-left: 20px" *ngIf="current_download.percent_complete && current_download.percent_complete > 1;else indeterminateprogress">
<mat-progress-bar style="border-radius: 5px;" mode="determinate" value="{{percentDownloaded}}"></mat-progress-bar>
<br/>
</div>

4
src/app/main/main.component.ts
View File

@@ -1033,8 +1033,8 @@ export class MainComponent implements OnInit {
}
} else if (format_obj.type === 'video') {
// check if video format is mp4
const key = format.height.toString();
if (format.ext === 'mp4') {
const key = format.format_note.replace('p', '');
if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') {
format_obj['height'] = format.height;
format_obj['acodec'] = format.acodec;
format_obj['format_id'] = format.format_id;

6
src/app/settings/settings.component.html
View File

@@ -128,7 +128,11 @@
</div>
<div class="col-12 mt-2">
<mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['safe_download_override']"><ng-container i18n="Safe download override setting">Safe download override</ng-container></mat-checkbox>
<mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['include_thumbnail']"><ng-container i18n="Include thumbnail setting">Include thumbnail</ng-container></mat-checkbox>
</div>
<div class="col-12 mt-2">
<mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['include_metadata']"><ng-container i18n="Include metadata setting">Include metadata</ng-container></mat-checkbox>
</div>
<div class="col-12 mt-2">