TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-16 09:40:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added the ability to download videos at higher resolutions than the highest mp4 (fixes #76)

Browse Source 
Deprecates normal downloading method. The "safe" method is now always used, and download progress is now estimated using the predicted end file size

Thumbnails are now auto downloaded along with the other metadata
This commit is contained in:
Isaac Abadi
2020-08-29 23:05:37 -04:00
parent 70159813e5
commit 8a7409478a
10 changed files with 174 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
backend/app.js
View File

@@ -7,7 +7,7 @@ var path = require('path');
var youtubedl = require('youtube-dl');
var ffmpeg = require('fluent-ffmpeg');
var compression = require('compression');
var https = require('https');
var glob = require("glob")
var multer = require('multer');
var express = require("express");
var bodyParser = require("body-parser");
@@ -1146,12 +1146,29 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now()
timestamp_start: Date.now(),
filesize: null
};
const download = downloads[session][download_uid];
updateDownloads();
// get video info prior to download
const info = await getVideoInfoByURL(url, downloadConfig, download);
if (!info) {
resolve(false);
return;
} else {
// store info in download for future use
download['_filename'] = info['_filename']; // .substring(fileFolderPath.length, info['_filename'].length-4);
download['filesize'] = utils.getExpectedFileSize(info);
}
const download_checker = setInterval(() => checkDownloadPercent(download), 1000);
// download file
youtubedl.exec(url, downloadConfig, {}, function(err, output) {
clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
download['downloading'] = false;
download['timestamp_end'] = Date.now();
var file_uid = null;
@@ -1164,7 +1181,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
download['error'] = err.stderr;
updateDownloads();
resolve(false);
throw err;
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
download['error'] = 'No output. Check if video already exists in your archive.';
@@ -1407,7 +1424,7 @@ async function generateArgs(url, type, options) {
var youtubePassword = options.youtubePassword;
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'best[ext=mp4]'];
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
@@ -1485,6 +1502,10 @@ async function generateArgs(url, type, options) {
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
@@ -1497,11 +1518,36 @@ async function generateArgs(url, type, options) {
}
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
// downloadConfig.map((arg) => `"${arg}"`);
resolve(downloadConfig);
});
}
async function getVideoInfoByURL(url, args = [], download = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
// actually get info
youtubedl.getInfo(url, new_args, (err, output) => {
if (output) {
resolve(output);
} else {
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
if (download) {
download['error'] = `Failed pre-check for video info: ${err}`;
updateDownloads();
}
resolve(null);
}
});
});
}
// currently only works for single urls
async function getUrlInfos(urls) {
let startDate = Date.now();
@@ -1559,47 +1605,26 @@ function updateDownloads() {
db.assign({downloads: downloads}).write();
}
/*
function checkDownloads() {
for (let [session_id, session_downloads] of Object.entries(downloads)) {
for (let [download_uid, download_obj] of Object.entries(session_downloads)) {
if (download_obj && !download_obj['complete'] && !download_obj['error']
&& download_obj.timestamp_start > timestamp_server_start) {
// download is still running (presumably)
download_obj.percent_complete = getDownloadPercent(download_obj);
}
}
}
}
*/
function checkDownloadPercent(download) {
const file_id = download['file_id'];
const filename = path.format(path.parse(download['_filename'].substring(0, download['_filename'].length-4)));
const resulting_file_size = download['filesize'];
function getDownloadPercent(download_obj) {
if (!download_obj.final_size) {
if (fs.existsSync(download_obj.expected_json_path)) {
const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8'));
let calculated_filesize = null;
if (file_json['format_id']) {
calculated_filesize = 0;
const formats_used = file_json['format_id'].split('+');
for (let i = 0; i < file_json['formats'].length; i++) {
if (formats_used.includes(file_json['formats'][i]['format_id'])) {
calculated_filesize += file_json['formats'][i]['filesize'];
}
glob(`${filename}*`, (err, files) => {
let sum_size = 0;
files.forEach(file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
download_obj.final_size = calculated_filesize;
} else {
console.log('could not find json file');
}
}
if (fs.existsSync(download_obj.expected_path)) {
const stats = fs.statSync(download_obj.expected_path);
const size = stats.size;
return (size / download_obj.final_size)*100;
} else {
console.log('could not find file');
return 0;
}
});
download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
updateDownloads();
});
}
// youtube-dl functions
@@ -1821,7 +1846,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
const is_playlist = url.includes('playlist');
let result_obj = null;
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
@@ -1833,6 +1858,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
});
app.post('/api/tomp4', optionalJwt, async function(req, res) {
req.setTimeout(0); // remove timeout in case of long videos
var url = req.body.url;
var options = {
customArgs: req.body.customArgs,
@@ -1850,7 +1876,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
const is_playlist = url.includes('playlist');
let result_obj = null;
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
@@ -1878,6 +1904,12 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
playlists = auth_api.getUserPlaylists(req.user.uid, 'audio');
}
// add thumbnails if present
mp3s.forEach(mp3 => {
if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath']))
mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']);
});
res.send({
mp3s: mp3s,
playlists: playlists
@@ -1897,6 +1929,12 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) {
playlists = auth_api.getUserPlaylists(req.user.uid, 'video');
}
// add thumbnails if present
mp4s.forEach(mp4 => {
if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath']))
mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']);
});
res.send({
mp4s: mp4s,
playlists: playlists
@@ -1981,6 +2019,12 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) {
files = files.concat(sub.videos);
}
// add thumbnails if present
files.forEach(file => {
if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath']))
file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']);
});
res.send({
files: files,
playlists: playlists

4
backend/appdata/default.json
View File

@@ -9,7 +9,9 @@
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",

4
backend/config.js
View File

@@ -186,7 +186,9 @@ DEFAULT_CONFIG = {
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": "",
"safe_download_override": false
"safe_download_override": false,
"include_thumbnail": true,
"include_metadata": true
},
"Extra": {
"title_top": "YoutubeDL-Material",

8
backend/consts.js
View File

@@ -30,6 +30,14 @@ let CONFIG_ITEMS = {
'key': 'ytdl_safe_download_override',
'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
},
'ytdl_include_thumbnail': {
'key': 'ytdl_include_thumbnail',
'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
},
'ytdl_include_metadata': {
'key': 'ytdl_include_metadata',
'path': 'YoutubeDLMaterial.Downloader.include_metadata'
},
// Extra
'ytdl_title_top': {

4
backend/db.js
View File

@@ -29,8 +29,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
// add additional info
file_object['uid'] = uuid();
file_object['registered'] = Date.now();
path_object = path.parse(file_object['path']);
file_object['path'] = path.format(path_object);
file_object['path'] = path.format(path.parse(file_object['path']));
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path);
if (!sub) {
if (multiUserMode) {

1
backend/package.json
View File

@@ -37,6 +37,7 @@
"express": "^4.17.1",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
"jsonwebtoken": "^8.5.1",
"lowdb": "^1.0.0",
"md5": "^2.2.1",

4
backend/subscriptions.js
View File

@@ -345,6 +345,10 @@ async function getVideosForSub(sub, user_uid = null) {
}
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {

37
backend/utils.js
View File

@@ -88,6 +88,41 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
}
function getDownloadedThumbnail(name, type, customPath = null) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getExpectedFileSize(info_json) {
if (info_json['filesize']) {
return info_json['filesize'];
}
const formats = info_json['format_id'].split('+');
let expected_filesize = 0;
formats.forEach(format_id => {
info_json.formats.forEach(available_format => {
if (available_format.format_id === format_id && available_format.filesize) {
expected_filesize += available_format.filesize;
}
});
});
return expected_filesize;
}
function fixVideoMetadataPerms(name, type, customPath = null) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
@@ -153,6 +188,8 @@ module.exports = {
getJSONMp3: getJSONMp3,
getJSONMp4: getJSONMp4,
getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail,
getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms,
getDownloadedFilesByType: getDownloadedFilesByType,
recFindByExt: recFindByExt,

2
src/app/components/unified-file-card/unified-file-card.component.html
View File

@@ -29,7 +29,7 @@
<div style="padding:5px">
<div *ngIf="!loading && file_obj.thumbnailURL" class="img-div">
<div style="position: relative">
<img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailURL" alt="Thumbnail">
<img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailBlob ? thumbnailBlobURL : file_obj.thumbnailURL" alt="Thumbnail">
<div class="duration-time">
{{file_length}}
</div>

27
src/app/components/unified-file-card/unified-file-card.component.ts
View File

@@ -1,6 +1,7 @@
import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core';
import { MatDialog } from '@angular/material/dialog';
import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component';
import { DomSanitizer } from '@angular/platform-browser';
@Component({
selector: 'app-unified-file-card',
@@ -16,6 +17,10 @@ export class UnifiedFileCardComponent implements OnInit {
type = null;
elevated = false;
// optional vars
thumbnailBlobURL = null;
// input/output
@Input() loading = true;
@Input() theme = null;
@Input() file_obj = null;
@@ -35,12 +40,19 @@ export class UnifiedFileCardComponent implements OnInit {
big: 250x200
*/
constructor(private dialog: MatDialog) { }
constructor(private dialog: MatDialog, private sanitizer: DomSanitizer) { }
ngOnInit(): void {
if (!this.loading) {
this.file_length = fancyTimeFormat(this.file_obj.duration);
}
if (this.file_obj && this.file_obj.thumbnailBlob) {
const mime = getMimeByFilename(this.file_obj.thumbnailPath);
const blob = new Blob([new Uint8Array(this.file_obj.thumbnailBlob.data)], {type: mime});
const bloburl = URL.createObjectURL(blob);
this.thumbnailBlobURL = this.sanitizer.bypassSecurityTrustUrl(bloburl);
}
}
emitDeleteFile(blacklistMode = false) {
@@ -97,3 +109,16 @@ function fancyTimeFormat(time) {
ret += '' + secs;
return ret;
}
function getMimeByFilename(name) {
switch (name.substring(name.length-4, name.length)) {
case '.jpg':
return 'image/jpeg';
case '.png':
return 'image/png';
case 'webp':
return 'image/webp';
default:
return null;
}
}