mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-07 12:00:01 +03:00
42c600cea9
Skipped tests just meant for manual testing mocha now used prod mode Added automated tests for PRs Fixed backupdb and youtube-dl tests in gh actions Fixed downloader tests Removed erroneous logs in tests Updated mocha.yml Removed last console.log in tests Increased timeout for download video test Automated tests now force stop after tests complete Replaced download test url with shorter video Skip download test... for now increased download test threshold added more timing information to tests unskip download test Bump docker/setup-buildx-action from 1 to 2 in /.github/workflows Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Bump actions/setup-node from 2 to 3 in /.github/workflows Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2 to 3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Bump actions/upload-artifact from 1 to 3 in /.github/workflows Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 1 to 3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v1...v3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Bump docker/login-action from 1 to 2 in /.github/workflows Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Removed additional download button for paused download Combined output json processing from youtube-dl between subscriptions and one-off downloads Added workaround for missing video in playlist to one-off downloads Fixed issue of case where output is empty Download video code cleanup Fixed download video test by overriding youtube-dl download method with custom method Adds ffmpeg to automated tests Removed test mp3/mp4 file from repo
627 lines
28 KiB
JavaScript
627 lines
28 KiB
JavaScript
const fs = require('fs-extra');
|
|
const { uuid } = require('uuidv4');
|
|
const path = require('path');
|
|
const NodeID3 = require('node-id3')
|
|
const Mutex = require('async-mutex').Mutex;
|
|
|
|
const youtubedl = require('youtube-dl');
|
|
|
|
const logger = require('./logger');
|
|
const youtubedl_api = require('./youtube-dl');
|
|
const config_api = require('./config');
|
|
const twitch_api = require('./twitch');
|
|
const { create } = require('xmlbuilder2');
|
|
const categories_api = require('./categories');
|
|
const utils = require('./utils');
|
|
const db_api = require('./db');
|
|
const files_api = require('./files');
|
|
const notifications_api = require('./notifications');
|
|
const archive_api = require('./archive');
|
|
|
|
const mutex = new Mutex();
|
|
let should_check_downloads = true;
|
|
|
|
if (db_api.database_initialized) {
|
|
exports.setupDownloads();
|
|
} else {
|
|
db_api.database_initialized_bs.subscribe(init => {
|
|
if (init) exports.setupDownloads();
|
|
});
|
|
}
|
|
|
|
/*
|
|
|
|
This file handles all the downloading functionality.
|
|
|
|
To download a file, we go through 4 steps. Here they are with their respective index & function:
|
|
|
|
0: Create the download
|
|
- createDownload()
|
|
1: Get info for the download (we need this step for categories and archive functionality)
|
|
- collectInfo()
|
|
2: Download the file
|
|
- downloadQueuedFile()
|
|
3: Complete
|
|
- N/A
|
|
|
|
We use checkDownloads() to move downloads through the steps and call their respective functions.
|
|
|
|
*/
|
|
|
|
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null, paused = false) => {
|
|
return await mutex.runExclusive(async () => {
|
|
const download = {
|
|
url: url,
|
|
type: type,
|
|
title: '',
|
|
user_uid: user_uid,
|
|
sub_id: sub_id,
|
|
sub_name: sub_name,
|
|
prefetched_info: prefetched_info,
|
|
options: options,
|
|
uid: uuid(),
|
|
step_index: 0,
|
|
paused: paused,
|
|
running: false,
|
|
finished_step: true,
|
|
error: null,
|
|
percent_complete: null,
|
|
finished: false,
|
|
timestamp_start: Date.now()
|
|
};
|
|
await db_api.insertRecordIntoTable('download_queue', download);
|
|
|
|
should_check_downloads = true;
|
|
return download;
|
|
});
|
|
}
|
|
|
|
exports.pauseDownload = async (download_uid) => {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
if (download['paused']) {
|
|
logger.warn(`Download ${download_uid} is already paused!`);
|
|
return false;
|
|
} else if (download['finished']) {
|
|
logger.info(`Download ${download_uid} could not be paused before completing.`);
|
|
return false;
|
|
}
|
|
|
|
return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false});
|
|
}
|
|
|
|
exports.resumeDownload = async (download_uid) => {
|
|
return await mutex.runExclusive(async () => {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
if (!download['paused']) {
|
|
logger.warn(`Download ${download_uid} is not paused!`);
|
|
return false;
|
|
}
|
|
|
|
const success = db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false});
|
|
should_check_downloads = true;
|
|
return success;
|
|
})
|
|
}
|
|
|
|
exports.restartDownload = async (download_uid) => {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
await exports.clearDownload(download_uid);
|
|
const new_download = await exports.createDownload(download['url'], download['type'], download['options'], download['user_uid']);
|
|
|
|
should_check_downloads = true;
|
|
return new_download;
|
|
}
|
|
|
|
exports.cancelDownload = async (download_uid) => {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
if (download['cancelled']) {
|
|
logger.warn(`Download ${download_uid} is already cancelled!`);
|
|
return false;
|
|
} else if (download['finished']) {
|
|
logger.info(`Download ${download_uid} could not be cancelled before completing.`);
|
|
return false;
|
|
}
|
|
return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false});
|
|
}
|
|
|
|
exports.clearDownload = async (download_uid) => {
|
|
return await db_api.removeRecord('download_queue', {uid: download_uid});
|
|
}
|
|
|
|
async function handleDownloadError(download, error_message, error_type = null) {
|
|
if (!download || !download['uid']) return;
|
|
notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type);
|
|
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
|
|
}
|
|
|
|
exports.setupDownloads = async () => {
|
|
await fixDownloadState();
|
|
setInterval(checkDownloads, 1000);
|
|
}
|
|
|
|
async function fixDownloadState() {
|
|
const downloads = await db_api.getRecords('download_queue');
|
|
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
|
|
const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
|
|
for (let i = 0; i < running_downloads.length; i++) {
|
|
const running_download = running_downloads[i];
|
|
const update_obj = {finished_step: true, paused: true, running: false};
|
|
if (running_download['step_index'] > 0) {
|
|
update_obj['step_index'] = running_download['step_index'] - 1;
|
|
}
|
|
await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj);
|
|
}
|
|
}
|
|
|
|
async function checkDownloads() {
|
|
if (!should_check_downloads) return;
|
|
|
|
const downloads = await db_api.getRecords('download_queue');
|
|
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
|
|
|
|
await mutex.runExclusive(async () => {
|
|
// avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
|
|
const running_downloads = downloads.filter(download => !download['paused'] && !download['finished']);
|
|
if (running_downloads.length === 0) {
|
|
should_check_downloads = false;
|
|
logger.verbose('Disabling checking downloads as none are available.');
|
|
}
|
|
return;
|
|
});
|
|
|
|
let running_downloads_count = downloads.filter(download => download['running']).length;
|
|
const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
|
|
for (let i = 0; i < waiting_downloads.length; i++) {
|
|
const waiting_download = waiting_downloads[i];
|
|
const max_concurrent_downloads = config_api.getConfigItem('ytdl_max_concurrent_downloads');
|
|
if (max_concurrent_downloads < 0 || running_downloads_count >= max_concurrent_downloads) break;
|
|
|
|
if (waiting_download['finished_step'] && !waiting_download['finished']) {
|
|
if (waiting_download['sub_id']) {
|
|
const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']}));
|
|
if (sub_missing) {
|
|
handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing');
|
|
continue;
|
|
}
|
|
}
|
|
// move to next step
|
|
running_downloads_count++;
|
|
if (waiting_download['step_index'] === 0) {
|
|
collectInfo(waiting_download['uid']);
|
|
} else if (waiting_download['step_index'] === 1) {
|
|
exports.downloadQueuedFile(waiting_download['uid']);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function collectInfo(download_uid) {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
if (download['paused']) {
|
|
return;
|
|
}
|
|
logger.verbose(`Collecting info for download ${download_uid}`);
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true});
|
|
|
|
const url = download['url'];
|
|
const type = download['type'];
|
|
const options = download['options'];
|
|
|
|
if (download['user_uid'] && !options.customFileFolderPath) {
|
|
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
|
const user_path = path.join(usersFileFolder, download['user_uid'], type);
|
|
options.customFileFolderPath = user_path + path.sep;
|
|
}
|
|
|
|
let args = await exports.generateArgs(url, type, options, download['user_uid']);
|
|
|
|
// get video info prior to download
|
|
let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid);
|
|
|
|
if (!info) {
|
|
// info failed, error presumably already recorded
|
|
return;
|
|
}
|
|
|
|
// in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
|
|
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
|
|
if (useYoutubeDLArchive && !options.ignoreArchive) {
|
|
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
|
|
if (exists_in_archive) {
|
|
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
|
|
logger.warn(error);
|
|
if (download_uid) {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
await handleDownloadError(download, error, 'exists_in_archive');
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
let category = null;
|
|
|
|
// check if it fits into a category. If so, then get info again using new args
|
|
if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
|
|
|
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
|
|
if (category && category['custom_output']) {
|
|
options.customOutput = category['custom_output'];
|
|
options.noRelativePath = true;
|
|
args = await exports.generateArgs(url, type, options, download['user_uid']);
|
|
info = await exports.getVideoInfoByURL(url, args, download_uid);
|
|
}
|
|
|
|
const stripped_category = category ? {name: category['name'], uid: category['uid']} : null;
|
|
|
|
// setup info required to calculate download progress
|
|
|
|
const expected_file_size = utils.getExpectedFileSize(info);
|
|
|
|
const files_to_check_for_progress = [];
|
|
|
|
// store info in download for future use
|
|
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
|
|
|
|
const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null;
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
|
|
finished_step: true,
|
|
running: false,
|
|
options: options,
|
|
files_to_check_for_progress: files_to_check_for_progress,
|
|
expected_file_size: expected_file_size,
|
|
title: playlist_title ? playlist_title : info['title'],
|
|
category: stripped_category,
|
|
prefetched_info: null
|
|
});
|
|
}
|
|
|
|
exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
if (download['paused']) {
|
|
return;
|
|
}
|
|
logger.verbose(`Downloading ${download_uid}`);
|
|
return new Promise(async resolve => {
|
|
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
|
|
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
|
|
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true});
|
|
|
|
const url = download['url'];
|
|
const type = download['type'];
|
|
const options = download['options'];
|
|
const args = download['args'];
|
|
const category = download['category'];
|
|
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
|
|
if (options.customFileFolderPath) {
|
|
fileFolderPath = options.customFileFolderPath;
|
|
} else if (download['user_uid']) {
|
|
fileFolderPath = path.join(usersFolderPath, download['user_uid'], type);
|
|
}
|
|
fs.ensureDirSync(fileFolderPath);
|
|
|
|
const start_time = Date.now();
|
|
|
|
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
|
|
const file_objs = [];
|
|
// download file
|
|
const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
|
|
clearInterval(download_checker);
|
|
let end_time = Date.now();
|
|
let difference = (end_time - start_time)/1000;
|
|
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
|
|
if (!parsed_output) {
|
|
logger.error(err.stderr);
|
|
await handleDownloadError(download, err.stderr, 'unknown_error');
|
|
resolve(false);
|
|
return;
|
|
} else if (parsed_output) {
|
|
if (parsed_output.length === 0 || parsed_output[0].length === 0) {
|
|
// ERROR!
|
|
const error_message = `No output received for video download, check if it exists in your archive.`;
|
|
await handleDownloadError(download, error_message, 'no_output');
|
|
logger.warn(error_message);
|
|
resolve(false);
|
|
return;
|
|
}
|
|
|
|
for (const output_json of parsed_output) {
|
|
if (!output_json) {
|
|
continue;
|
|
}
|
|
|
|
// get filepath with no extension
|
|
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
|
|
|
|
const ext = type === 'audio' ? '.mp3' : '.mp4';
|
|
var full_file_path = filepath_no_extension + ext;
|
|
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
|
|
|
|
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
|
|
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
|
|
let vodId = url.split('twitch.tv/videos/')[1];
|
|
vodId = vodId.split('?')[0];
|
|
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
|
|
}
|
|
|
|
// renames file if necessary due to bug
|
|
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
|
|
try {
|
|
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
|
|
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
|
|
} catch(e) {
|
|
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
|
|
}
|
|
}
|
|
|
|
if (type === 'audio') {
|
|
let tags = {
|
|
title: output_json['title'],
|
|
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
|
|
}
|
|
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
|
|
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
|
|
}
|
|
|
|
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
|
|
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
|
|
}
|
|
|
|
if (options.cropFileSettings) {
|
|
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
|
|
}
|
|
|
|
// registers file in DB
|
|
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
|
|
|
|
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
|
|
|
|
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
|
|
|
|
file_objs.push(file_obj);
|
|
}
|
|
|
|
let container = null;
|
|
|
|
if (file_objs.length > 1) {
|
|
// create playlist
|
|
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
|
|
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
|
|
} else if (file_objs.length === 1) {
|
|
container = file_objs[0];
|
|
} else {
|
|
const error_message = 'Downloaded file failed to result in metadata object.';
|
|
logger.error(error_message);
|
|
await handleDownloadError(download, error_message, 'no_metadata');
|
|
}
|
|
|
|
const file_uids = file_objs.map(file_obj => file_obj.uid);
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
|
|
resolve(file_uids);
|
|
}
|
|
});
|
|
}
|
|
|
|
// helper functions
|
|
|
|
exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => {
|
|
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
|
|
|
|
if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) {
|
|
logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.')
|
|
}
|
|
|
|
const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
|
|
const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
|
|
const usersFolderPath = config_api.getConfigItem('ytdl_users_base_path');
|
|
|
|
const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
|
|
const globalArgs = config_api.getConfigItem('ytdl_custom_args');
|
|
const useCookies = config_api.getConfigItem('ytdl_use_cookies');
|
|
const is_audio = type === 'audio';
|
|
|
|
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
|
|
if (options.customFileFolderPath) {
|
|
fileFolderPath = options.customFileFolderPath;
|
|
} else if (user_uid) {
|
|
fileFolderPath = path.join(usersFolderPath, user_uid, fileFolderPath);
|
|
}
|
|
|
|
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
|
|
|
|
const customArgs = options.customArgs;
|
|
let customOutput = options.customOutput;
|
|
const customQualityConfiguration = options.customQualityConfiguration;
|
|
|
|
// video-specific args
|
|
const selectedHeight = options.selectedHeight;
|
|
const maxHeight = options.maxHeight;
|
|
const heightParam = selectedHeight || maxHeight;
|
|
|
|
// audio-specific args
|
|
const maxBitrate = options.maxBitrate;
|
|
|
|
const youtubeUsername = options.youtubeUsername;
|
|
const youtubePassword = options.youtubePassword;
|
|
|
|
let downloadConfig = null;
|
|
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
|
|
const is_youtube = url.includes('youtu');
|
|
if (!is_audio && !is_youtube) {
|
|
// tiktok videos fail when using the default format
|
|
qualityPath = null;
|
|
}
|
|
|
|
if (customArgs) {
|
|
downloadConfig = customArgs.split(',,');
|
|
} else {
|
|
if (customQualityConfiguration) {
|
|
qualityPath = ['-f', customQualityConfiguration, '--merge-output-format', 'mp4'];
|
|
} else if (heightParam && heightParam !== '' && !is_audio) {
|
|
const heightFilter = (maxHeight && default_downloader === 'yt-dlp') ? ['-S', `res:${heightParam}`] : ['-f', `best[height${maxHeight ? '<' : ''}=${heightParam}]+bestaudio`]
|
|
qualityPath = [...heightFilter, '--merge-output-format', 'mp4'];
|
|
} else if (is_audio) {
|
|
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
|
|
}
|
|
|
|
if (customOutput) {
|
|
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
|
|
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
|
|
} else {
|
|
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
|
|
}
|
|
|
|
if (qualityPath) downloadConfig.push(...qualityPath);
|
|
|
|
if (is_audio && !options.skip_audio_args) {
|
|
downloadConfig.push('-x');
|
|
downloadConfig.push('--audio-format', 'mp3');
|
|
}
|
|
|
|
if (youtubeUsername && youtubePassword) {
|
|
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
|
|
}
|
|
|
|
if (useCookies) {
|
|
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
|
|
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
|
|
} else {
|
|
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
|
|
}
|
|
}
|
|
|
|
const useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
|
|
const customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
|
|
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
|
|
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
|
|
}
|
|
|
|
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
|
|
downloadConfig.push('--write-thumbnail');
|
|
}
|
|
|
|
if (globalArgs && globalArgs !== '') {
|
|
// adds global args
|
|
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
|
|
// if global args has an output, replce the original output with that of global args
|
|
const original_output_index = downloadConfig.indexOf('-o');
|
|
downloadConfig.splice(original_output_index, 2);
|
|
}
|
|
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
|
|
}
|
|
|
|
if (options.additionalArgs && options.additionalArgs !== '') {
|
|
downloadConfig = utils.injectArgs(downloadConfig, options.additionalArgs.split(',,'));
|
|
}
|
|
|
|
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
|
|
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
|
|
downloadConfig.push('-r', rate_limit);
|
|
}
|
|
|
|
if (default_downloader === 'yt-dlp') {
|
|
downloadConfig = utils.filterArgs(downloadConfig, ['--print-json']);
|
|
|
|
// in yt-dlp -j --no-simulate is preferable
|
|
downloadConfig.push('--no-clean-info-json', '-j', '--no-simulate');
|
|
}
|
|
|
|
}
|
|
|
|
// filter out incompatible args
|
|
downloadConfig = filterArgs(downloadConfig, is_audio);
|
|
|
|
if (!simulated) logger.verbose(`${default_downloader} args being used: ${downloadConfig.join(',')}`);
|
|
return downloadConfig;
|
|
}
|
|
|
|
exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
|
|
return new Promise(resolve => {
|
|
// remove bad args
|
|
const temp_args = utils.filterArgs(args, ['--no-simulate']);
|
|
const new_args = [...temp_args];
|
|
|
|
const archiveArgIndex = new_args.indexOf('--download-archive');
|
|
if (archiveArgIndex !== -1) {
|
|
new_args.splice(archiveArgIndex, 2);
|
|
}
|
|
|
|
new_args.push('--dump-json');
|
|
|
|
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
|
|
const parsed_output = utils.parseOutputJSON(output, err);
|
|
if (parsed_output) {
|
|
resolve(parsed_output);
|
|
} else {
|
|
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
|
|
if (err.stderr) error_message += `\n\n${err.stderr}`;
|
|
logger.error(error_message);
|
|
if (download_uid) {
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
await handleDownloadError(download, error_message, 'info_retrieve_failed');
|
|
}
|
|
resolve(null);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
function filterArgs(args, isAudio) {
|
|
const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
|
|
const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
|
|
return utils.filterArgs(args, isAudio ? video_only_args : audio_only_args);
|
|
}
|
|
|
|
async function checkDownloadPercent(download_uid) {
|
|
/*
|
|
This is more of an art than a science, we're just selecting files that start with the file name,
|
|
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
|
|
|
|
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
|
|
be divided by the "total expected bytes."
|
|
*/
|
|
|
|
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
|
const files_to_check_for_progress = download['files_to_check_for_progress'];
|
|
const resulting_file_size = download['expected_file_size'];
|
|
|
|
if (!resulting_file_size) return;
|
|
|
|
let sum_size = 0;
|
|
for (let i = 0; i < files_to_check_for_progress.length; i++) {
|
|
const file_to_check_for_progress = files_to_check_for_progress[i];
|
|
const dir = path.dirname(file_to_check_for_progress);
|
|
if (!fs.existsSync(dir)) continue;
|
|
fs.readdir(dir, async (err, files) => {
|
|
for (let j = 0; j < files.length; j++) {
|
|
const file = files[j];
|
|
if (!file.includes(path.basename(file_to_check_for_progress))) continue;
|
|
try {
|
|
const file_stats = fs.statSync(path.join(dir, file));
|
|
if (file_stats && file_stats.size) {
|
|
sum_size += file_stats.size;
|
|
}
|
|
} catch (e) {}
|
|
}
|
|
|
|
const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
|
|
});
|
|
}
|
|
}
|
|
|
|
exports.generateNFOFile = (info, output_path) => {
|
|
const nfo_obj = {
|
|
episodedetails: {
|
|
title: info['fulltitle'],
|
|
episode: info['playlist_index'] ? info['playlist_index'] : undefined,
|
|
premiered: utils.formatDateString(info['upload_date']),
|
|
plot: `${info['uploader_url']}\n${info['description']}\n${info['playlist_title'] ? info['playlist_title'] : ''}`,
|
|
director: info['artist'] ? info['artist'] : info['uploader']
|
|
}
|
|
};
|
|
const doc = create(nfo_obj);
|
|
const xml = doc.end({ prettyPrint: true });
|
|
fs.writeFileSync(output_path, xml);
|
|
}
|