mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-29 16:10:56 +03:00
355 lines
15 KiB
JavaScript
355 lines
15 KiB
JavaScript
const fs = require('fs-extra');
|
|
const { uuid } = require('uuidv4');
|
|
const path = require('path');
|
|
const queue = require('queue');
|
|
|
|
const youtubedl = require('youtube-dl');
|
|
const config_api = require('./config');
|
|
const twitch_api = require('./twitch');
|
|
const utils = require('./utils');
|
|
|
|
let db_api = null;
|
|
let logger = null;
|
|
|
|
function setDB(input_db_api) { db_api = input_db_api }
|
|
function setLogger(input_logger) { logger = input_logger; }
|
|
|
|
exports.initialize = (input_db_api, input_logger) => {
|
|
setDB(input_db_api);
|
|
setLogger(input_logger);
|
|
}
|
|
|
|
exports.pauseDownload = () => {
|
|
|
|
}
|
|
|
|
async function checkDownloads() {
|
|
const downloads = await db_api.getRecords('download_queue');
|
|
downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
|
|
downloads = downloads.filter(download => !download.paused);
|
|
for (let i = 0; i < downloads.length; i++) {
|
|
if (i === config_api.getConfigItem('ytdl_'))
|
|
}
|
|
}
|
|
|
|
async function createDownload(url, type, options) {
|
|
const download = {url: url, type: type, options: options, uid: uuid()};
|
|
await db_api.insertRecord(download);
|
|
return download;
|
|
}
|
|
|
|
async function collectInfo(download_uid) {
|
|
const download = db_api.getRecord('download_queue', {uid: download_uid});
|
|
|
|
const url = download['url'];
|
|
const type = download['type'];
|
|
const options = download['options'];
|
|
const args = download['args'];
|
|
|
|
// get video info prior to download
|
|
const info = await getVideoInfoByURL(url, args);
|
|
|
|
if (!info) {
|
|
// info failed, record error and pause download
|
|
}
|
|
|
|
// check if it fits into a category. If so, then get info again using new args
|
|
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
|
|
|
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
|
|
if (category && category['custom_output']) {
|
|
options.customOutput = category['custom_output'];
|
|
options.noRelativePath = true;
|
|
args = await generateArgs(url, type, options);
|
|
info = await getVideoInfoByURL(url, args);
|
|
|
|
// must update args
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args});
|
|
}
|
|
|
|
await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info});
|
|
}
|
|
|
|
async function downloadQueuedFile(url, type, options) {
|
|
|
|
}
|
|
|
|
async function downloadFileByURL_exec(url, type, options) {
|
|
return new Promise(resolve => {
|
|
const download = db_api.getRecord('download_queue', {uid: download_uid});
|
|
|
|
const url = download['url'];
|
|
const type = download['type'];
|
|
const options = download['options'];
|
|
const args = download['args'];
|
|
const category = download['category'];
|
|
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
|
|
if (options.user) {
|
|
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
|
const user_path = path.join(usersFileFolder, options.user, type);
|
|
fs.ensureDirSync(user_path);
|
|
fileFolderPath = user_path + path.sep;
|
|
options.customFileFolderPath = fileFolderPath;
|
|
}
|
|
|
|
// download file
|
|
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
|
|
const file_objs = [];
|
|
let new_date = Date.now();
|
|
let difference = (new_date - date)/1000;
|
|
logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
|
|
if (err) {
|
|
logger.error(err.stderr);
|
|
|
|
resolve(false);
|
|
return;
|
|
} else if (output) {
|
|
if (output.length === 0 || output[0].length === 0) {
|
|
// ERROR!
|
|
logger.warn(`No output received for video download, check if it exists in your archive.`)
|
|
|
|
resolve(false);
|
|
return;
|
|
}
|
|
|
|
for (let i = 0; i < output.length; i++) {
|
|
let output_json = null;
|
|
try {
|
|
output_json = JSON.parse(output[i]);
|
|
} catch(e) {
|
|
output_json = null;
|
|
}
|
|
|
|
if (!output_json) {
|
|
continue;
|
|
}
|
|
|
|
// get filepath with no extension
|
|
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
|
|
|
|
var full_file_path = filepath_no_extension + ext;
|
|
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
|
|
|
|
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
|
|
&& config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) {
|
|
let vodId = url.split('twitch.tv/videos/')[1];
|
|
vodId = vodId.split('?')[0];
|
|
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user);
|
|
}
|
|
|
|
// renames file if necessary due to bug
|
|
if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
|
|
try {
|
|
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
|
|
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
|
|
} catch(e) {
|
|
}
|
|
}
|
|
|
|
if (type === 'audio') {
|
|
let tags = {
|
|
title: output_json['title'],
|
|
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
|
|
}
|
|
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
|
|
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
|
|
}
|
|
|
|
if (options.cropFileSettings) {
|
|
await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
|
|
}
|
|
|
|
// registers file in DB
|
|
const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings);
|
|
|
|
file_objs.push(file_obj);
|
|
}
|
|
|
|
if (options.merged_string !== null && options.merged_string !== undefined) {
|
|
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
|
|
let diff = current_merged_archive.replace(options.merged_string, '');
|
|
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
|
|
fs.appendFileSync(archive_path, diff);
|
|
}
|
|
|
|
let container = null;
|
|
|
|
if (file_objs.length > 1) {
|
|
// create playlist
|
|
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
|
|
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user);
|
|
} else if (file_objs.length === 1) {
|
|
container = file_objs[0];
|
|
} else {
|
|
logger.error('Downloaded file failed to result in metadata object.');
|
|
}
|
|
|
|
resolve({
|
|
file_uids: file_objs.map(file_obj => file_obj.uid),
|
|
container: container
|
|
});
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
// helper functions
|
|
|
|
async function generateArgs(url, type, options) {
|
|
const videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
|
|
const globalArgs = config_api.getConfigItem('ytdl_custom_args');
|
|
const useCookies = config_api.getConfigItem('ytdl_use_cookies');
|
|
const is_audio = type === 'audio';
|
|
|
|
const fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
|
|
|
|
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
|
|
|
|
const customArgs = options.customArgs;
|
|
const customOutput = options.customOutput;
|
|
const customQualityConfiguration = options.customQualityConfiguration;
|
|
|
|
// video-specific args
|
|
const selectedHeight = options.selectedHeight;
|
|
|
|
// audio-specific args
|
|
const maxBitrate = options.maxBitrate;
|
|
|
|
const youtubeUsername = options.youtubeUsername;
|
|
const youtubePassword = options.youtubePassword;
|
|
|
|
let downloadConfig = null;
|
|
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
|
|
const is_youtube = url.includes('youtu');
|
|
if (!is_audio && !is_youtube) {
|
|
// tiktok videos fail when using the default format
|
|
qualityPath = null;
|
|
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
|
|
qualityPath = ['-f', 'bestvideo+bestaudio']
|
|
}
|
|
|
|
if (customArgs) {
|
|
downloadConfig = customArgs.split(',,');
|
|
} else {
|
|
if (customQualityConfiguration) {
|
|
qualityPath = ['-f', customQualityConfiguration];
|
|
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
|
|
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
|
|
} else if (is_audio) {
|
|
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
|
|
}
|
|
|
|
if (customOutput) {
|
|
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
|
|
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
|
|
} else {
|
|
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
|
|
}
|
|
|
|
if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath);
|
|
|
|
if (is_audio && !options.skip_audio_args) {
|
|
downloadConfig.push('-x');
|
|
downloadConfig.push('--audio-format', 'mp3');
|
|
}
|
|
|
|
if (youtubeUsername && youtubePassword) {
|
|
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
|
|
}
|
|
|
|
if (useCookies) {
|
|
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
|
|
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
|
|
} else {
|
|
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
|
|
}
|
|
}
|
|
|
|
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
|
|
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
|
|
}
|
|
|
|
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
|
|
if (useYoutubeDLArchive) {
|
|
const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath;
|
|
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
|
|
|
|
await fs.ensureDir(archive_folder);
|
|
await fs.ensureFile(archive_path);
|
|
|
|
let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
|
|
await fs.ensureFile(blacklist_path);
|
|
|
|
let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
|
|
await fs.ensureFile(merged_path);
|
|
// merges blacklist and regular archive
|
|
let inputPathList = [archive_path, blacklist_path];
|
|
let status = await mergeFiles(inputPathList, merged_path);
|
|
|
|
options.merged_string = await fs.readFile(merged_path, "utf8");
|
|
|
|
downloadConfig.push('--download-archive', merged_path);
|
|
}
|
|
|
|
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
|
|
downloadConfig.push('--write-thumbnail');
|
|
}
|
|
|
|
if (globalArgs && globalArgs !== '') {
|
|
// adds global args
|
|
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
|
|
// if global args has an output, replce the original output with that of global args
|
|
const original_output_index = downloadConfig.indexOf('-o');
|
|
downloadConfig.splice(original_output_index, 2);
|
|
}
|
|
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
|
|
}
|
|
|
|
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
|
|
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
|
|
downloadConfig.push('-r', rate_limit);
|
|
}
|
|
|
|
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
|
|
if (default_downloader === 'yt-dlp') {
|
|
downloadConfig.push('--no-clean-infojson');
|
|
}
|
|
|
|
}
|
|
|
|
// filter out incompatible args
|
|
downloadConfig = filterArgs(downloadConfig, is_audio);
|
|
|
|
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
|
|
return downloadConfig;
|
|
}
|
|
|
|
async function getVideoInfoByURL(url, args = [], download = null) {
|
|
return new Promise(resolve => {
|
|
// remove bad args
|
|
const new_args = [...args];
|
|
|
|
const archiveArgIndex = new_args.indexOf('--download-archive');
|
|
if (archiveArgIndex !== -1) {
|
|
new_args.splice(archiveArgIndex, 2);
|
|
}
|
|
|
|
// actually get info
|
|
youtubedl.getInfo(url, new_args, (err, output) => {
|
|
if (output) {
|
|
resolve(output);
|
|
} else {
|
|
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
|
|
if (err.stderr) {
|
|
logger.error(`${err.stderr}`)
|
|
}
|
|
if (download) {
|
|
download['error'] = `Failed pre-check for video info: ${err}`;
|
|
updateDownloads();
|
|
}
|
|
resolve(null);
|
|
}
|
|
});
|
|
});
|
|
}
|