From d887380fd18102086bb5f4c3cb002f545c9cd5db Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Sat, 18 Apr 2020 01:31:32 -0400 Subject: [PATCH 1/7] Added new methods to facilitate server-side download management --- backend/app.js | 35 +++++++++++++++++++++++++++++++++++ src/app/posts.services.ts | 5 +++++ 2 files changed, 40 insertions(+) diff --git a/backend/app.js b/backend/app.js index a8a7c82..d8a4c0d 100644 --- a/backend/app.js +++ b/backend/app.js @@ -141,6 +141,7 @@ if (writeConfigMode) { loadConfig(); } +var downloads = {}; var descriptors = {}; app.use(bodyParser.urlencoded({ extended: false })); @@ -1363,7 +1364,18 @@ app.post('/api/tomp3', async function(req, res) { } } + // adds download to download helper + const download_uid = uuid(); + downloads[download_uid] = { + uid: download_uid, + downloading: true, + complete: false, + url: url, + type: 'audio' + }; + youtubedl.exec(url, downloadConfig, {}, function(err, output) { + downloads[download_uid]['downloading'] = false; var uid = null; let new_date = Date.now(); let difference = (new_date - date)/1000; @@ -1423,6 +1435,8 @@ app.post('/api/tomp3', async function(req, res) { fs.unlinkSync(merged_path) } + downloads[download_uid]['complete'] = true; + var audiopathEncoded = encodeURIComponent(file_names[0]); res.send({ audiopathEncoded: audiopathEncoded, @@ -1510,7 +1524,20 @@ app.post('/api/tomp4', async function(req, res) { } + // adds download to download helper + const download_uid = uuid(); + downloads[download_uid] = { + uid: download_uid, + downloading: true, + complete: false, + url: url, + type: 'video', + percent_complete: 0, + is_playlist: url.includes('playlist') + }; + youtubedl.exec(url, downloadConfig, {}, function(err, output) { + downloads[download_uid]['downloading'] = false; var uid = null; let new_date = Date.now(); let difference = (new_date - date)/1000; @@ -1568,6 +1595,8 @@ app.post('/api/tomp4', async function(req, res) { const archive_path = path.join(archivePath, 'archive_video.txt'); fs.appendFileSync(archive_path, diff); } + + downloads[download_uid]['complete'] = true; var videopathEncoded = encodeURIComponent(file_names[0]); res.send({ @@ -2282,6 +2311,12 @@ app.get('/api/audio/:id', function(req , res){ } }); + // Downloads management + + app.get('/api/downloads', async (req, res) => { + res.send({downloads: downloads}); + }); + app.post('/api/getVideoInfos', async (req, res) => { let fileNames = req.body.fileNames; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 8aa05d1..6112e75 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -215,6 +215,11 @@ export class PostsService { return this.http.post(this.path + 'getAllSubscriptions', {}, this.httpOptions); } + // current downloads + getCurrentDownloads() { + return this.http.get(this.path + 'downloads', this.httpOptions); + } + // updates the server to the latest version updateServer(tag) { return this.http.post(this.path + 'updateServer', {tag: tag}, this.httpOptions); From 6fe7d20498286e9e16bca798da84432f431709dc Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Mon, 20 Apr 2020 18:39:55 -0400 Subject: [PATCH 2/7] downloads refactor half done - videos are now implement, but audo files are now Added downloads manager in the UI where downloads can be viewed/cleared --- backend/app.js | 588 +++++++++++++----- src/app/app-routing.module.ts | 2 + src/app/app.component.html | 5 +- src/app/app.module.ts | 4 +- .../downloads/downloads.component.html | 24 + .../downloads/downloads.component.scss | 0 .../downloads/downloads.component.spec.ts | 25 + .../downloads/downloads.component.ts | 106 ++++ .../download-item.component.html | 21 +- .../download-item.component.scss | 16 + .../download-item/download-item.component.ts | 4 +- src/app/main/main.component.ts | 11 +- src/app/posts.services.ts | 16 +- 13 files changed, 644 insertions(+), 178 deletions(-) create mode 100644 src/app/components/downloads/downloads.component.html create mode 100644 src/app/components/downloads/downloads.component.scss create mode 100644 src/app/components/downloads/downloads.component.spec.ts create mode 100644 src/app/components/downloads/downloads.component.ts diff --git a/backend/app.js b/backend/app.js index d8a4c0d..5787999 100644 --- a/backend/app.js +++ b/backend/app.js @@ -77,6 +77,7 @@ db.defaults( video: [] }, configWriteFlag: false, + downloads: {}, subscriptions: [], pin_md5: '', files_to_db_migration_complete: false @@ -101,6 +102,10 @@ var archivePath = path.join(__dirname, 'appdata', 'archives'); var options = null; // encryption options var url_domain = null; var updaterStatus = null; +var last_downloads_check = null; +var downloads_check_interval = 1000; + +var timestamp_server_start = Date.now(); if (debugMode) logger.info('YTDL-Material in debug mode!'); @@ -234,7 +239,6 @@ async function startServer() { logger.info(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on PORT ${backendPort}`); }); } - } async function restartServer() { @@ -547,6 +551,9 @@ async function loadConfig() { // check migrations await checkMigrations(); + // load in previous downloads + downloads = db.get('downloads').value(); + // start the server here startServer(); @@ -999,6 +1006,7 @@ function registerFileDB(full_file_path, type) { } file_object['uid'] = uuid(); + file_object['registered'] = Date.now(); path_object = path.parse(file_object['path']); file_object['path'] = path.format(path_object); db.get(`files.${type}`) @@ -1081,6 +1089,304 @@ function getVideoInfos(fileNames) { return result; } +// downloads + +async function downloadFileByURL_exec(url, type, options, sessionID = null) { + return new Promise(async resolve => { + var date = Date.now(); + + const downloadConfig = await generateArgs(url, type, options); + + // adds download to download helper + const download_uid = uuid(); + const session = sessionID ? sessionID : 'undeclared'; + if (!downloads[session]) downloads[session] = {}; + downloads[session][download_uid] = { + uid: download_uid, + downloading: true, + complete: false, + url: url, + type: type, + percent_complete: 0, + is_playlist: url.includes('playlist'), + timestamp_start: Date.now() + }; + const download = downloads[session][download_uid]; + updateDownloads(); + + await new Promise(resolve => { + youtubedl.exec(url, [...downloadConfig, '--dump-json'], {}, function(err, output) { + if (output) { + let json = JSON.parse(output[0]); + const output_no_ext = removeFileExtension(json['_filename']); + download['expected_path'] = output_no_ext + '.mp4'; + download['expected_json_path'] = output_no_ext + '.info.json'; + resolve(true); + } else if (err) { + logger.error(err.stderr); + } else { + logger.error(`Video info retrieval failed. Download progress will be unavailable for URL ${url}`); + } + + }); + }); + youtubedl.exec(url, downloadConfig, {}, function(err, output) { + download['downloading'] = false; + download['timestamp_end'] = Date.now(); + var file_uid = null; + let new_date = Date.now(); + let difference = (new_date - date)/1000; + logger.debug(`Video download delay: ${difference} seconds.`); + if (err) { + logger.error(err.stderr); + + download['error'] = err.stderr; + updateDownloads(); + resolve(false); + throw err; + } else if (output) { + if (output.length === 0 || output[0].length === 0) { + download['error'] = 'No output. Check if video already exists in your archive.'; + updateDownloads(); + + resolve(false); + return; + } + var file_names = []; + for (let i = 0; i < output.length; i++) { + let output_json = null; + try { + output_json = JSON.parse(output[i]); + } catch(e) { + output_json = null; + } + var modified_file_name = output_json ? output_json['title'] : null; + if (!output_json) { + continue; + } + + // get filepath with no extension + const filepath_no_extension = removeFileExtension(output_json['_filename']); + + var full_file_path = filepath_no_extension + '.mp4'; + var file_name = filepath_no_extension.substring(audioFolderPath.length, filepath_no_extension.length); + + // renames file if necessary due to bug + if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { + try { + fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); + logger.info('Renamed ' + file_name + '.webm to ' + file_name); + } catch(e) { + } + } + + // registers file in DB + file_uid = registerFileDB(full_file_path.substring(videoFolderPath.length, full_file_path.length), 'video'); + + if (file_name) file_names.push(file_name); + } + + let is_playlist = file_names.length > 1; + + if (options.merged_string !== null) { + let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); + let diff = current_merged_archive.replace(options.merged_string, ''); + const archive_path = path.join(archivePath, 'archive_video.txt'); + fs.appendFileSync(archive_path, diff); + } + + download['complete'] = true; + updateDownloads(); + + var videopathEncoded = encodeURIComponent(file_names[0]); + + resolve({ + videopathEncoded: videopathEncoded, + file_names: is_playlist ? file_names : null, + uid: file_uid + }); + } + }); + }); +} + +async function downloadFileByURL_normal(url, type, options, sessionID = null) { + return new Promise(async resolve => { + var date = Date.now(); + var file_uid = null; + + const downloadConfig = await generateArgs(url, type, options); + + // adds download to download helper + const download_uid = uuid(); + const session = sessionID ? sessionID : 'undeclared'; + if (!downloads[session]) downloads[session] = {}; + downloads[session][download_uid] = { + uid: download_uid, + downloading: true, + complete: false, + url: url, + type: type, + percent_complete: 0, + is_playlist: url.includes('playlist'), + timestamp_start: Date.now() + }; + const download = downloads[session][download_uid]; + updateDownloads(); + + const video = youtubedl(url, + // Optional arguments passed to youtube-dl. + downloadConfig, + // Additional options can be given for calling `child_process.execFile()`. + { cwd: __dirname }); + + let video_info = null; + let file_size = 0; + + // Will be called when the download starts. + video.on('info', function(info) { + video_info = info; + file_size = video_info.size; + console.log('Download started') + fs.writeJSONSync(removeFileExtension(video_info._filename) + '.info.json', video_info); + video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) + }); + // Will be called if download was already completed and there is nothing more to download. + video.on('complete', function complete(info) { + 'use strict' + console.log('filename: ' + info._filename + ' already downloaded.') + }) + + let download_pos = 0; + video.on('data', function data(chunk) { + download_pos += chunk.length + // `size` should not be 0 here. + if (file_size) { + let percent = (download_pos / file_size * 100).toFixed(2) + download['percent_complete'] = percent; + } + }); + + video.on('end', function() { + console.log('finished downloading!') + + let new_date = Date.now(); + let difference = (new_date - date)/1000; + logger.debug(`Video download delay: ${difference} seconds.`); + + download['complete'] = true; + updateDownloads(); + + // registers file in DB + const base_file_name = video_info._filename.substring(videoFolderPath.length, video_info._filename.length); + file_uid = registerFileDB(base_file_name, type); + + if (options.merged_string) { + let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); + let diff = current_merged_archive.replace(options.merged_string, ''); + const archive_path = path.join(archivePath, 'archive_video.txt'); + fs.appendFileSync(archive_path, diff); + } + + videopathEncoded = encodeURIComponent(removeFileExtension(base_file_name)); + + resolve({ + videopathEncoded: videopathEncoded, + file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready + uid: file_uid + }); + }); + + video.on('error', function error(err) { + logger.error(err); + + download[error] = err; + updateDownloads(); + + resolve(false); + }); + }); + +} + +async function generateArgs(url, type, options) { + return new Promise(async resolve => { + var videopath = '%(title)s'; + var globalArgs = config_api.getConfigItem('ytdl_custom_args'); + + var customArgs = options.customArgs; + var customOutput = options.customOutput; + + var selectedHeight = options.selectedHeight; + var customQualityConfiguration = options.customQualityConfiguration; + var youtubeUsername = options.youtubeUsername; + var youtubePassword = options.youtubePassword; + + let downloadConfig = null; + let qualityPath = 'best[ext=mp4]'; + + if (url.includes('tiktok') || url.includes('pscp.tv')) { + // tiktok videos fail when using the default format + qualityPath = 'best'; + } + + if (customArgs) { + downloadConfig = customArgs.split(' '); + } else { + if (customQualityConfiguration) { + qualityPath = customQualityConfiguration; + } else if (selectedHeight && selectedHeight !== '') { + qualityPath = `bestvideo[height=${selectedHeight}]+bestaudio/best[height=${selectedHeight}]`; + } + + if (customOutput) { + downloadConfig = ['-o', videoFolderPath + customOutput + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; + } else { + downloadConfig = ['-o', videoFolderPath + videopath + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; + } + + if (youtubeUsername && youtubePassword) { + downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); + } + + if (!useDefaultDownloadingAgent && customDownloadingAgent) { + downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); + } + + let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive) { + const archive_path = path.join(archivePath, 'archive_video.txt'); + // create archive file if it doesn't exist + if (!fs.existsSync(archive_path)) { + fs.closeSync(fs.openSync(archive_path, 'w')); + } + + let blacklist_path = path.join(archivePath, 'blacklist_video.txt'); + // create blacklist file if it doesn't exist + if (!fs.existsSync(blacklist_path)) { + fs.closeSync(fs.openSync(blacklist_path, 'w')); + } + + let merged_path = videoFolderPath + 'merged.txt'; + // merges blacklist and regular archive + let inputPathList = [archive_path, blacklist_path]; + let status = await mergeFiles(inputPathList, merged_path); + + options.merged_string = fs.readFileSync(merged_path, "utf8"); + + downloadConfig.push('--download-archive', merged_path); + } + + if (globalArgs && globalArgs !== '') { + // adds global args + downloadConfig = downloadConfig.concat(globalArgs.split(' ')); + } + + } + resolve(downloadConfig); + }); +} + // currently only works for single urls async function getUrlInfos(urls) { let startDate = Date.now(); @@ -1117,6 +1423,57 @@ function writeToBlacklist(type, line) { fs.appendFileSync(blacklistPath, line); } +// download management functions + +function updateDownloads() { + db.assign({downloads: downloads}).write(); +} + +/* +function checkDownloads() { + for (let [session_id, session_downloads] of Object.entries(downloads)) { + for (let [download_uid, download_obj] of Object.entries(session_downloads)) { + if (download_obj && !download_obj['complete'] && !download_obj['error'] + && download_obj.timestamp_start > timestamp_server_start) { + // download is still running (presumably) + download_obj.percent_complete = getDownloadPercent(download_obj); + } + } + } +} +*/ + +function getDownloadPercent(download_obj) { + if (!download_obj.final_size) { + if (fs.existsSync(download_obj.expected_json_path)) { + const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8')); + let calculated_filesize = null; + if (file_json['format_id']) { + calculated_filesize = 0; + const formats_used = file_json['format_id'].split('+'); + for (let i = 0; i < file_json['formats'].length; i++) { + if (formats_used.includes(file_json['formats'][i]['format_id'])) { + calculated_filesize += file_json['formats'][i]['filesize']; + } + } + } + download_obj.final_size = calculated_filesize; + } else { + console.log('could not find json file'); + } + } + if (fs.existsSync(download_obj.expected_path)) { + const stats = fs.statSync(download_obj.expected_path); + const size = stats.size; + return (size / download_obj.final_size)*100; + } else { + console.log('could not find file'); + return 0; + } +} + +// youtube-dl functions + async function startYoutubeDL() { // auto update youtube-dl await autoUpdateYoutubeDL(); @@ -1366,28 +1723,39 @@ app.post('/api/tomp3', async function(req, res) { // adds download to download helper const download_uid = uuid(); - downloads[download_uid] = { + const session = req.query.sessionID ? req.query.sessionID : 'undeclared'; + if (!downloads[session]) downloads[session] = {}; + downloads[session][download_uid] = { uid: download_uid, downloading: true, complete: false, url: url, - type: 'audio' + type: 'audio', + percent_complete: 0, + is_playlist: url.includes('playlist'), + timestamp_start: Date.now() }; - + updateDownloads(); youtubedl.exec(url, downloadConfig, {}, function(err, output) { - downloads[download_uid]['downloading'] = false; + downloads[session][download_uid]['downloading'] = false; var uid = null; let new_date = Date.now(); let difference = (new_date - date)/1000; logger.debug(`Audio download delay: ${difference} seconds.`); if (err) { - audiopath = "-1"; logger.error(err.stderr); + + downloads[session][download_uid]['error'] = err.stderr; + updateDownloads(); + res.sendStatus(500); throw err; } else if (output) { var file_names = []; if (output.length === 0 || output[0].length === 0) { + downloads[session][download_uid]['error'] = 'No output. Check if video already exists in your archive.'; + updateDownloads(); + res.sendStatus(500); return; } @@ -1435,7 +1803,8 @@ app.post('/api/tomp3', async function(req, res) { fs.unlinkSync(merged_path) } - downloads[download_uid]['complete'] = true; + downloads[session][download_uid]['complete'] = true; + updateDownloads(); var audiopathEncoded = encodeURIComponent(file_names[0]); res.send({ @@ -1449,164 +1818,23 @@ app.post('/api/tomp3', async function(req, res) { app.post('/api/tomp4', async function(req, res) { var url = req.body.url; - var date = Date.now(); - var videopath = '%(title)s'; - var globalArgs = config_api.getConfigItem('ytdl_custom_args'); - var customArgs = req.body.customArgs; - var customOutput = req.body.customOutput; - - var selectedHeight = req.body.selectedHeight; - var customQualityConfiguration = req.body.customQualityConfiguration; - var youtubeUsername = req.body.youtubeUsername; - var youtubePassword = req.body.youtubePassword; - - let merged_string = null; - - let downloadConfig = null; - let qualityPath = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'; - - if (url.includes('tiktok') || url.includes('pscp.tv')) { - // tiktok videos fail when using the default format - qualityPath = 'best'; + var options = { + customArgs: req.body.customArgs, + customOutput: req.body.customOutput, + selectedHeight: req.body.selectedHeight, + customQualityConfiguration: req.body.customQualityConfiguration, + youtubeUsername: req.body.youtubeUsername, + youtubePassword: req.body.youtubePassword } - if (customArgs) { - downloadConfig = customArgs.split(' '); + const result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID) + if (result_obj) { + res.send(result_obj); } else { - if (customQualityConfiguration) { - qualityPath = customQualityConfiguration; - } else if (selectedHeight && selectedHeight !== '') { - qualityPath = `bestvideo[height=${selectedHeight}]+bestaudio/best[height=${selectedHeight}]`; - } - - if (customOutput) { - downloadConfig = ['-o', videoFolderPath + customOutput + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; - } else { - downloadConfig = ['-o', videoFolderPath + videopath + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; - } - - if (youtubeUsername && youtubePassword) { - downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); - } - - if (!useDefaultDownloadingAgent && customDownloadingAgent) { - downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); - // create archive file if it doesn't exist - if (!fs.existsSync(archive_path)) { - fs.closeSync(fs.openSync(archive_path, 'w')); - } - - let blacklist_path = path.join(archivePath, 'blacklist_video.txt'); - // create blacklist file if it doesn't exist - if (!fs.existsSync(blacklist_path)) { - fs.closeSync(fs.openSync(blacklist_path, 'w')); - } - - let merged_path = videoFolderPath + 'merged.txt'; - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - let status = await mergeFiles(inputPathList, merged_path); - - merged_string = fs.readFileSync(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - - if (globalArgs && globalArgs !== '') { - // adds global args - downloadConfig = downloadConfig.concat(globalArgs.split(' ')); - } - + res.sendStatus(500); } - - // adds download to download helper - const download_uid = uuid(); - downloads[download_uid] = { - uid: download_uid, - downloading: true, - complete: false, - url: url, - type: 'video', - percent_complete: 0, - is_playlist: url.includes('playlist') - }; - - youtubedl.exec(url, downloadConfig, {}, function(err, output) { - downloads[download_uid]['downloading'] = false; - var uid = null; - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); - if (err) { - videopath = "-1"; - logger.error(err.stderr); - res.sendStatus(500); - throw err; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { - res.sendStatus(500); - return; - } - var file_names = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - var modified_file_name = output_json ? output_json['title'] : null; - if (!output_json) { - continue; - } - - // get filepath with no extension - const filepath_no_extension = removeFileExtension(output_json['_filename']); - - var full_file_path = filepath_no_extension + '.mp4'; - var file_name = filepath_no_extension.substring(audioFolderPath.length, filepath_no_extension.length); - - // renames file if necessary due to bug - if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { - try { - fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); - logger.info('Renamed ' + file_name + '.webm to ' + file_name); - } catch(e) { - } - } - - // registers file in DB - uid = registerFileDB(full_file_path.substring(videoFolderPath.length, full_file_path.length), 'video'); - - if (file_name) file_names.push(file_name); - } - - let is_playlist = file_names.length > 1; - if (!is_playlist) audiopath = file_names[0]; - - if (merged_string !== null) { - let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); - let diff = current_merged_archive.replace(merged_string, ''); - const archive_path = path.join(archivePath, 'archive_video.txt'); - fs.appendFileSync(archive_path, diff); - } - - downloads[download_uid]['complete'] = true; - - var videopathEncoded = encodeURIComponent(file_names[0]); - res.send({ - videopathEncoded: videopathEncoded, - file_names: is_playlist ? file_names : null, - uid: uid - }); - res.end("yes"); - } - }); + + res.end("yes"); }); // gets the status of the mp3 file that's being downloaded @@ -2314,9 +2542,47 @@ app.get('/api/audio/:id', function(req , res){ // Downloads management app.get('/api/downloads', async (req, res) => { + /* + if (!last_downloads_check || Date.now() - last_downloads_check > downloads_check_interval) { + last_downloads_check = Date.now(); + updateDownloads(); + } + */ res.send({downloads: downloads}); }); + app.post('/api/clearDownloads', async (req, res) => { + let success = false; + var delete_all = req.body.delete_all; + if (!req.body.session_id) req.body.session_id = 'undeclared'; + var session_id = req.body.session_id; + var download_id = req.body.download_id; + if (delete_all) { + // delete all downloads + downloads = {}; + success = true; + } else if (download_id) { + // delete just 1 download + if (downloads[session_id][download_id]) { + delete downloads[session_id][download_id]; + success = true; + } else if (!downloads[session_id]) { + logger.error(`Session ${session_id} has no downloads.`) + } else if (!downloads[session_id][download_id]) { + logger.error(`Download '${download_id}' for session '${session_id}' could not be found`); + } + } else if (session_id) { + // delete a session's downloads + if (downloads[session_id]) { + delete downloads[session_id]; + success = true; + } else { + logger.error(`Session ${session_id} has no downloads.`) + } + } + updateDownloads(); + res.send({success: success, downloads: downloads}); + }); app.post('/api/getVideoInfos', async (req, res) => { let fileNames = req.body.fileNames; diff --git a/src/app/app-routing.module.ts b/src/app/app-routing.module.ts index c4cce48..e1b3dcd 100644 --- a/src/app/app-routing.module.ts +++ b/src/app/app-routing.module.ts @@ -4,11 +4,13 @@ import { MainComponent } from './main/main.component'; import { PlayerComponent } from './player/player.component'; import { SubscriptionsComponent } from './subscriptions/subscriptions.component'; import { SubscriptionComponent } from './subscription/subscription/subscription.component'; +import { DownloadsComponent } from './components/downloads/downloads.component'; const routes: Routes = [ { path: 'home', component: MainComponent }, { path: 'player', component: PlayerComponent}, { path: 'subscriptions', component: SubscriptionsComponent }, { path: 'subscription', component: SubscriptionComponent }, + { path: 'downloads', component: DownloadsComponent }, { path: '', redirectTo: '/home', pathMatch: 'full' }, ]; diff --git a/src/app/app.component.html b/src/app/app.component.html index 2a0463f..0e03fe9 100644 --- a/src/app/app.component.html +++ b/src/app/app.component.html @@ -3,7 +3,7 @@
- +
@@ -37,7 +37,8 @@ Home - Subscriptions + Subscriptions + Downloads diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 6f03650..beff3cc 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -57,6 +57,7 @@ import { ArgModifierDialogComponent, HighlightPipe } from './dialogs/arg-modifie import { UpdaterComponent } from './updater/updater.component'; import { UpdateProgressDialogComponent } from './dialogs/update-progress-dialog/update-progress-dialog.component'; import { ShareMediaDialogComponent } from './dialogs/share-media-dialog/share-media-dialog.component'; +import { DownloadsComponent } from './components/downloads/downloads.component'; registerLocaleData(es, 'es'); export function isVisible({ event, element, scrollContainer, offset }: IsVisibleProps) { @@ -85,7 +86,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible HighlightPipe, UpdaterComponent, UpdateProgressDialogComponent, - ShareMediaDialogComponent + ShareMediaDialogComponent, + DownloadsComponent ], imports: [ CommonModule, diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html new file mode 100644 index 0000000..251ca9b --- /dev/null +++ b/src/app/components/downloads/downloads.component.html @@ -0,0 +1,24 @@ +
+
+ + +

Session ID: {{session_downloads.key}} +  (current) +

+
+
+
+ + + +
+
+
+ + +
+ +
+

No downloads available!

+
+
\ No newline at end of file diff --git a/src/app/components/downloads/downloads.component.scss b/src/app/components/downloads/downloads.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/src/app/components/downloads/downloads.component.spec.ts b/src/app/components/downloads/downloads.component.spec.ts new file mode 100644 index 0000000..e7a1fa6 --- /dev/null +++ b/src/app/components/downloads/downloads.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { DownloadsComponent } from './downloads.component'; + +describe('DownloadsComponent', () => { + let component: DownloadsComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ DownloadsComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(DownloadsComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts new file mode 100644 index 0000000..f07d8d7 --- /dev/null +++ b/src/app/components/downloads/downloads.component.ts @@ -0,0 +1,106 @@ +import { Component, OnInit, ViewChildren, QueryList, ElementRef } from '@angular/core'; +import { PostsService } from 'app/posts.services'; +import { trigger, transition, animateChild, stagger, query, style, animate } from '@angular/animations'; + +@Component({ + selector: 'app-downloads', + templateUrl: './downloads.component.html', + styleUrls: ['./downloads.component.scss'], + animations: [ + // nice stagger effect when showing existing elements + trigger('list', [ + transition(':enter', [ + // child animation selector + stagger + query('@items', + stagger(100, animateChild()), { optional: true } + ) + ]), + ]), + trigger('items', [ + // cubic-bezier for a tiny bouncing feel + transition(':enter', [ + style({ transform: 'scale(0.5)', opacity: 0 }), + animate('500ms cubic-bezier(.8,-0.6,0.2,1.5)', + style({ transform: 'scale(1)', opacity: 1 })) + ]), + transition(':leave', [ + style({ transform: 'scale(1)', opacity: 1, height: '*' }), + animate('1s cubic-bezier(.8,-0.6,0.2,1.5)', + style({ transform: 'scale(0.5)', opacity: 0, height: '0px', margin: '0px' })) + ]), + ]) + ], +}) +export class DownloadsComponent implements OnInit { + + downloads_check_interval = 500; + downloads = null; + + keys = Object.keys; + + valid_sessions_length = 0; + + constructor(public postsService: PostsService) { } + + ngOnInit(): void { + this.getCurrentDownloads(); + setInterval(() => { + this.getCurrentDownloads(); + }, this.downloads_check_interval); + } + + getCurrentDownloads() { + this.postsService.getCurrentDownloads().subscribe(res => { + if (res['downloads']) { + if (JSON.stringify(this.downloads) !== JSON.stringify(res['downloads'])) { + // if they're not the same, then replace + this.downloads = res['downloads']; + } + } else { + // failed to get downloads + } + }); + } + + clearDownload(session_id, download_uid) { + this.postsService.clearDownloads(false, session_id, download_uid).subscribe(res => { + if (res['success']) { + this.downloads = res['downloads']; + } else { + } + }); + } + + clearDownloads(session_id) { + this.postsService.clearDownloads(false, session_id).subscribe(res => { + if (res['success']) { + this.downloads = res['downloads']; + } else { + } + }); + } + + clearAllDownloads() { + this.postsService.clearDownloads(true).subscribe(res => { + if (res['success']) { + this.downloads = res['downloads']; + } else { + } + }); + } + + downloadsValid() { + let valid = false; + const keys = this.keys(this.downloads); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const value = this.downloads[key]; + if (this.keys(value).length > 0) { + valid = true; + break; + } + } + return valid; + } + +} diff --git a/src/app/download-item/download-item.component.html b/src/app/download-item/download-item.component.html index 5c5a02f..cc112c8 100644 --- a/src/app/download-item/download-item.component.html +++ b/src/app/download-item/download-item.component.html @@ -1,16 +1,21 @@
- -
{{queueNumber}}.
- - -
ID: {{url_id}}
+ +
ID: {{url_id ? url_id : download.uid}}
 - + + done + error - - + + + + + Error + + {{download.error}} +
\ No newline at end of file diff --git a/src/app/download-item/download-item.component.scss b/src/app/download-item/download-item.component.scss index e69de29..f28a49f 100644 --- a/src/app/download-item/download-item.component.scss +++ b/src/app/download-item/download-item.component.scss @@ -0,0 +1,16 @@ +.shorten { + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + display: block; +} + +.mat-expansion-panel:not([class*='mat-elevation-z']) { + box-shadow: none; +} + +.ignore-margin { + margin-left: -15px; + margin-right: -15px; + margin-bottom: -15px; +} diff --git a/src/app/download-item/download-item.component.ts b/src/app/download-item/download-item.component.ts index 6795d11..5520ace 100644 --- a/src/app/download-item/download-item.component.ts +++ b/src/app/download-item/download-item.component.ts @@ -13,9 +13,11 @@ export class DownloadItemComponent implements OnInit { uid: null, type: 'audio', percent_complete: 0, + complete: false, url: 'http://youtube.com/watch?v=17848rufj', downloading: true, - is_playlist: false + is_playlist: false, + error: false }; @Output() cancelDownload = new EventEmitter(); diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 568b1ce..69d4813 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -34,7 +34,9 @@ export interface Download { percent_complete: number; downloading: boolean; is_playlist: boolean; + error: boolean | string; fileNames?: string[]; + complete?: boolean; } @Component({ @@ -207,7 +209,8 @@ export class MainComponent implements OnInit { percent_complete: 0, url: 'http://youtube.com/watch?v=17848rufj', downloading: true, - is_playlist: false + is_playlist: false, + error: false }; simulatedOutput = ''; @@ -571,7 +574,8 @@ export class MainComponent implements OnInit { percent_complete: 0, url: this.url, downloading: true, - is_playlist: this.url.includes('playlist') + is_playlist: this.url.includes('playlist'), + error: false }; this.downloads.push(new_download); if (!this.current_download && !this.multiDownloadMode) { this.current_download = new_download }; @@ -613,7 +617,8 @@ export class MainComponent implements OnInit { percent_complete: 0, url: this.url, downloading: true, - is_playlist: this.url.includes('playlist') + is_playlist: this.url.includes('playlist'), + error: false }; this.downloads.push(new_download); if (!this.current_download && !this.multiDownloadMode) { this.current_download = new_download }; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 6112e75..a34de62 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -8,6 +8,7 @@ import { THEMES_CONFIG } from '../themes'; import { Router } from '@angular/router'; import { DOCUMENT } from '@angular/common'; import { BehaviorSubject } from 'rxjs'; +import { v4 as uuid } from 'uuid'; @Injectable() export class PostsService { @@ -21,7 +22,9 @@ export class PostsService { theme; settings_changed = new BehaviorSubject(false); auth_token = '4241b401-7236-493e-92b5-b72696b9d853'; + session_id = null; httpOptions = null; + http_params: string = null; debugMode = false; constructor(private http: HttpClient, private router: Router, @Inject(DOCUMENT) private document: Document) { @@ -29,15 +32,17 @@ export class PostsService { // this.startPath = window.location.href + '/api/'; // this.startPathSSL = window.location.href + '/api/'; this.path = this.document.location.origin + '/api/'; - + this.session_id = uuid(); if (isDevMode()) { this.debugMode = true; this.path = 'http://localhost:17442/api/'; } + this.http_params = `apiKey=${this.auth_token}&sessionID=${this.session_id}` + this.httpOptions = { params: new HttpParams({ - fromString: `apiKey=${this.auth_token}` + fromString: this.http_params }), }; } @@ -220,6 +225,13 @@ export class PostsService { return this.http.get(this.path + 'downloads', this.httpOptions); } + // clear downloads. download_id is optional, if it exists only 1 download will be cleared + clearDownloads(delete_all = false, session_id = null, download_id = null) { + return this.http.post(this.path + 'clearDownloads', {delete_all: delete_all, + download_id: download_id, + session_id: session_id ? session_id : this.session_id}, this.httpOptions); + } + // updates the server to the latest version updateServer(tag) { return this.http.post(this.path + 'updateServer', {tag: tag}, this.httpOptions); From a6534f66a630d1f4eaa13f8965ad27a60c17e3a5 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Tue, 21 Apr 2020 03:16:39 -0400 Subject: [PATCH 3/7] migrated audio file downloads to new system. still untested with playlists video/audio player now doesnt show share button when uid isn't present, user will be notified of this through a snackbar as well --- backend/app.js | 268 ++++++++------------------- src/app/player/player.component.html | 2 +- src/app/player/player.component.ts | 4 + 3 files changed, 83 insertions(+), 191 deletions(-) diff --git a/backend/app.js b/backend/app.js index 5787999..c7528a3 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1094,6 +1094,11 @@ function getVideoInfos(fileNames) { async function downloadFileByURL_exec(url, type, options, sessionID = null) { return new Promise(async resolve => { var date = Date.now(); + + // audio / video specific vars + var is_audio = type === 'audio'; + var ext = is_audio ? '.mp3' : '.mp4'; + var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; const downloadConfig = await generateArgs(url, type, options); @@ -1119,7 +1124,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { if (output) { let json = JSON.parse(output[0]); const output_no_ext = removeFileExtension(json['_filename']); - download['expected_path'] = output_no_ext + '.mp4'; + download['expected_path'] = output_no_ext + ext; download['expected_json_path'] = output_no_ext + '.info.json'; resolve(true); } else if (err) { @@ -1168,8 +1173,8 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // get filepath with no extension const filepath_no_extension = removeFileExtension(output_json['_filename']); - var full_file_path = filepath_no_extension + '.mp4'; - var file_name = filepath_no_extension.substring(audioFolderPath.length, filepath_no_extension.length); + var full_file_path = filepath_no_extension + ext; + var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); // renames file if necessary due to bug if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { @@ -1180,18 +1185,27 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } } + if (type === 'audio') { + let tags = { + title: output_json['title'], + artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] + } + let success = NodeID3.write(tags, output_json['_filename']); + if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); + } + // registers file in DB - file_uid = registerFileDB(full_file_path.substring(videoFolderPath.length, full_file_path.length), 'video'); + file_uid = registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), 'video'); if (file_name) file_names.push(file_name); } let is_playlist = file_names.length > 1; - if (options.merged_string !== null) { - let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); + if (options.merged_string) { + let current_merged_archive = fs.readFileSync(fileFolderPath + 'merged.txt', 'utf8'); let diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = path.join(archivePath, 'archive_video.txt'); + const archive_path = path.join(archivePath, `archive_${type}.txt`); fs.appendFileSync(archive_path, diff); } @@ -1201,7 +1215,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { var videopathEncoded = encodeURIComponent(file_names[0]); resolve({ - videopathEncoded: videopathEncoded, + [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, file_names: is_playlist ? file_names : null, uid: file_uid }); @@ -1214,6 +1228,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { return new Promise(async resolve => { var date = Date.now(); var file_uid = null; + var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; const downloadConfig = await generateArgs(url, type, options); @@ -1277,12 +1292,22 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { download['complete'] = true; updateDownloads(); + // Does ID3 tagging if audio + if (type === 'audio') { + let tags = { + title: video_info['title'], + artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] + } + let success = NodeID3.write(tags, video_info._filename); + if (!success) logger.error('Failed to apply ID3 tag to audio file ' + video_info._filename); + } + // registers file in DB - const base_file_name = video_info._filename.substring(videoFolderPath.length, video_info._filename.length); + const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); file_uid = registerFileDB(base_file_name, type); if (options.merged_string) { - let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); + let current_merged_archive = fs.readFileSync(fileFolderPath + 'merged.txt', 'utf8'); let diff = current_merged_archive.replace(options.merged_string, ''); const archive_path = path.join(archivePath, 'archive_video.txt'); fs.appendFileSync(archive_path, diff); @@ -1291,7 +1316,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { videopathEncoded = encodeURIComponent(removeFileExtension(base_file_name)); resolve({ - videopathEncoded: videopathEncoded, + [(type === 'audio') ? 'audiopathEncoded' : 'videopathEncoded']: videopathEncoded, file_names: /*is_playlist ? file_names :*/ null, // playlist support is not ready uid: file_uid }); @@ -1313,21 +1338,29 @@ async function generateArgs(url, type, options) { return new Promise(async resolve => { var videopath = '%(title)s'; var globalArgs = config_api.getConfigItem('ytdl_custom_args'); + var is_audio = type === 'audio'; + + var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath; var customArgs = options.customArgs; var customOutput = options.customOutput; - - var selectedHeight = options.selectedHeight; var customQualityConfiguration = options.customQualityConfiguration; + + // video-specific args + var selectedHeight = options.selectedHeight; + + // audio-specific args + var maxBitrate = options.maxBitrate; + var youtubeUsername = options.youtubeUsername; var youtubePassword = options.youtubePassword; let downloadConfig = null; - let qualityPath = 'best[ext=mp4]'; + let qualityPath = is_audio ? '-f bestaudio' :'-f best[ext=mp4]'; - if (url.includes('tiktok') || url.includes('pscp.tv')) { + if (!is_audio && (url.includes('tiktok') || url.includes('pscp.tv'))) { // tiktok videos fail when using the default format - qualityPath = 'best'; + qualityPath = '-f best'; } if (customArgs) { @@ -1335,14 +1368,20 @@ async function generateArgs(url, type, options) { } else { if (customQualityConfiguration) { qualityPath = customQualityConfiguration; - } else if (selectedHeight && selectedHeight !== '') { - qualityPath = `bestvideo[height=${selectedHeight}]+bestaudio/best[height=${selectedHeight}]`; + } else if (selectedHeight && selectedHeight !== '' && !is_audio) { + qualityPath = `-f bestvideo[height=${selectedHeight}]+bestaudio/best[height=${selectedHeight}]`; + } else if (maxBitrate && is_audio) { + qualityPath = `--audio-quality ${maxBitrate}` } if (customOutput) { - downloadConfig = ['-o', videoFolderPath + customOutput + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; + downloadConfig = ['-o', fileFolderPath + customOutput + "", qualityPath, '--write-info-json', '--print-json']; } else { - downloadConfig = ['-o', videoFolderPath + videopath + ".mp4", '-f', qualityPath, '--write-info-json', '--print-json']; + downloadConfig = ['-o', fileFolderPath + videopath + (is_audio ? '.mp3' : '.mp4'), qualityPath, '--write-info-json', '--print-json']; + } + + if (is_audio) { + downloadConfig.push('--audio-format', 'mp3'); } if (youtubeUsername && youtubePassword) { @@ -1355,19 +1394,20 @@ async function generateArgs(url, type, options) { let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_video.txt'); + const archive_path = path.join(archivePath, `archive_${type}.txt`); // create archive file if it doesn't exist if (!fs.existsSync(archive_path)) { fs.closeSync(fs.openSync(archive_path, 'w')); } - let blacklist_path = path.join(archivePath, 'blacklist_video.txt'); + let blacklist_path = path.join(archivePath, `blacklist_${type}.txt`); // create blacklist file if it doesn't exist if (!fs.existsSync(blacklist_path)) { fs.closeSync(fs.openSync(blacklist_path, 'w')); } - let merged_path = videoFolderPath + 'merged.txt'; + let merged_path = fileFolderPath + 'merged.txt'; + fs.ensureFileSync(merged_path); // merges blacklist and regular archive let inputPathList = [archive_path, blacklist_path]; let status = await mergeFiles(inputPathList, merged_path); @@ -1645,175 +1685,23 @@ app.get('/api/using-encryption', function(req, res) { app.post('/api/tomp3', async function(req, res) { var url = req.body.url; - var date = Date.now(); - var audiopath = '%(title)s'; - - var customQualityConfiguration = req.body.customQualityConfiguration; - var maxBitrate = req.body.maxBitrate; - var globalArgs = config_api.getConfigItem('ytdl_custom_args'); - var customArgs = req.body.customArgs; - var customOutput = req.body.customOutput; - var youtubeUsername = req.body.youtubeUsername; - var youtubePassword = req.body.youtubePassword; - - let downloadConfig = null; - let qualityPath = '-f bestaudio'; - - let merged_path = null; - let merged_string = null; - - if (customArgs) { - downloadConfig = customArgs.split(' '); - } else { - if (customQualityConfiguration) { - qualityPath = `-f ${customQualityConfiguration}`; - } else if (maxBitrate) { - if (!maxBitrate || maxBitrate === '') maxBitrate = '0'; - qualityPath = `--audio-quality ${maxBitrate}` - } - - if (customOutput) { - downloadConfig = ['-x', '--audio-format', 'mp3', '-o', audioFolderPath + customOutput + '.%(ext)s', '--write-info-json', '--print-json']; - } else { - downloadConfig = ['-x', '--audio-format', 'mp3', '-o', audioFolderPath + audiopath + ".%(ext)s", '--write-info-json', '--print-json']; - } - - if (youtubeUsername && youtubePassword) { - downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword); - } - - if (qualityPath !== '') { - downloadConfig.splice(3, 0, qualityPath); - } - - if (!useDefaultDownloadingAgent && customDownloadingAgent) { - downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_path = path.join(archivePath, 'archive_audio.txt'); - // create archive file if it doesn't exist - if (!fs.existsSync(archive_path)) { - fs.closeSync(fs.openSync(archive_path, 'w')); - } - - let blacklist_path = path.join(archivePath, 'blacklist_audio.txt'); - // create blacklist file if it doesn't exist - if (!fs.existsSync(blacklist_path)) { - fs.closeSync(fs.openSync(blacklist_path, 'w')); - } - - // creates merged folder - merged_path = audioFolderPath + `merged_${uuid()}.txt`; - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - let status = await mergeFiles(inputPathList, merged_path); - - merged_string = fs.readFileSync(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - - if (globalArgs && globalArgs !== '') { - // adds global args - downloadConfig = downloadConfig.concat(globalArgs.split(' ')); - } + var options = { + customArgs: req.body.customArgs, + customOutput: req.body.customOutput, + maxBitrate: req.body.maxBitrate, + customQualityConfiguration: req.body.customQualityConfiguration, + youtubeUsername: req.body.youtubeUsername, + youtubePassword: req.body.youtubePassword } - // adds download to download helper - const download_uid = uuid(); - const session = req.query.sessionID ? req.query.sessionID : 'undeclared'; - if (!downloads[session]) downloads[session] = {}; - downloads[session][download_uid] = { - uid: download_uid, - downloading: true, - complete: false, - url: url, - type: 'audio', - percent_complete: 0, - is_playlist: url.includes('playlist'), - timestamp_start: Date.now() - }; - updateDownloads(); - youtubedl.exec(url, downloadConfig, {}, function(err, output) { - downloads[session][download_uid]['downloading'] = false; - var uid = null; - let new_date = Date.now(); - let difference = (new_date - date)/1000; - logger.debug(`Audio download delay: ${difference} seconds.`); - if (err) { - logger.error(err.stderr); - - downloads[session][download_uid]['error'] = err.stderr; - updateDownloads(); - - res.sendStatus(500); - throw err; - } else if (output) { - var file_names = []; - if (output.length === 0 || output[0].length === 0) { - downloads[session][download_uid]['error'] = 'No output. Check if video already exists in your archive.'; - updateDownloads(); - - res.sendStatus(500); - return; - } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - if (!output_json) { - // if invalid, continue onto the next - continue; - } - - const filepath_no_extension = removeFileExtension(output_json['_filename']); - - var full_file_path = filepath_no_extension + '.mp3'; - var file_name = filepath_no_extension.substring(audioFolderPath.length, filepath_no_extension.length); - if (fs.existsSync(full_file_path)) { - let tags = { - title: output_json['title'], - artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] - } - // NodeID3.create(tags, function(frame) { }) - let success = NodeID3.write(tags, full_file_path); - if (!success) logger.error('Failed to apply ID3 tag to audio file ' + full_file_path); - - // registers file in DB - uid = registerFileDB(full_file_path.substring(audioFolderPath.length, full_file_path.length), 'audio'); - } else { - logger.error('Download failed: Output mp3 does not exist'); - } - - if (file_name) file_names.push(file_name); - } - - let is_playlist = file_names.length > 1; - - if (merged_string !== null) { - let current_merged_archive = fs.readFileSync(merged_path, 'utf8'); - let diff = current_merged_archive.replace(merged_string, ''); - const archive_path = path.join(archivePath, 'archive_audio.txt'); - fs.appendFileSync(archive_path, diff); - fs.unlinkSync(merged_path) - } - - downloads[session][download_uid]['complete'] = true; - updateDownloads(); - - var audiopathEncoded = encodeURIComponent(file_names[0]); - res.send({ - audiopathEncoded: audiopathEncoded, - file_names: is_playlist ? file_names : null, - uid: uid - }); - } - }); + const result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID) + if (result_obj) { + res.send(result_obj); + } else { + res.sendStatus(500); + } + + res.end("yes"); }); app.post('/api/tomp4', async function(req, res) { diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 4c9e294..6c03c3e 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -30,6 +30,6 @@
- +
\ No newline at end of file diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index e8b819f..83acebb 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -125,6 +125,10 @@ export class PlayerComponent implements OnInit { const already_has_filenames = !!this.fileNames; this.postsService.getFile(this.uid, null).subscribe(res => { this.db_file = res['file']; + if (!this.db_file) { + this.openSnackBar('Failed to get file information from the server.', 'Dismiss'); + return; + } if (!this.fileNames) { // means it's a shared video if (!this.id) { From 1565c328d5dc51deaa8da97511386971c8879d41 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Tue, 21 Apr 2020 16:19:19 -0400 Subject: [PATCH 4/7] If a video is a playlist, it will download the normal way --- backend/app.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/app.js b/backend/app.js index c7528a3..f99a563 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1694,7 +1694,11 @@ app.post('/api/tomp3', async function(req, res) { youtubePassword: req.body.youtubePassword } - const result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID) + const is_playlist = url.includes('playlist'); + if (is_playlist) + result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID); + else + result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID); if (result_obj) { res.send(result_obj); } else { @@ -1714,8 +1718,13 @@ app.post('/api/tomp4', async function(req, res) { youtubeUsername: req.body.youtubeUsername, youtubePassword: req.body.youtubePassword } - - const result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID) + + const is_playlist = url.includes('playlist'); + let result_obj = null; + if (is_playlist) + result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID); + else + result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID); if (result_obj) { res.send(result_obj); } else { From f361b8a974bd4b6b58decec119d16d96e5a7a8c6 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Tue, 21 Apr 2020 18:56:52 -0400 Subject: [PATCH 5/7] Furrther simplified download process and fixed a couple bugs Audio files will not show download progress as enabling this feature causes it to be really slow Fixed bug where downloading the same video twice produced duplicate files in the file manager --- backend/app.js | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/backend/app.js b/backend/app.js index f99a563..ef87782 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1005,10 +1005,18 @@ function registerFileDB(full_file_path, type) { return false; } + // add additional info file_object['uid'] = uuid(); file_object['registered'] = Date.now(); path_object = path.parse(file_object['path']); file_object['path'] = path.format(path_object); + + // remove existing video if overwriting + db.get(`files.${type}`) + .remove({ + path: file_object['path'] + }).write(); + db.get(`files.${type}`) .push(file_object) .write(); @@ -1022,6 +1030,7 @@ function generateFileObject(id, type) { } const ext = (type === 'audio') ? '.mp3' : '.mp4' const file_path = getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); + // console. var stats = fs.statSync(path.join(__dirname, file_path)); var title = jsonobj.title; @@ -1117,31 +1126,15 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { timestamp_start: Date.now() }; const download = downloads[session][download_uid]; - updateDownloads(); + updateDownloads(); - await new Promise(resolve => { - youtubedl.exec(url, [...downloadConfig, '--dump-json'], {}, function(err, output) { - if (output) { - let json = JSON.parse(output[0]); - const output_no_ext = removeFileExtension(json['_filename']); - download['expected_path'] = output_no_ext + ext; - download['expected_json_path'] = output_no_ext + '.info.json'; - resolve(true); - } else if (err) { - logger.error(err.stderr); - } else { - logger.error(`Video info retrieval failed. Download progress will be unavailable for URL ${url}`); - } - - }); - }); youtubedl.exec(url, downloadConfig, {}, function(err, output) { download['downloading'] = false; download['timestamp_end'] = Date.now(); var file_uid = null; let new_date = Date.now(); let difference = (new_date - date)/1000; - logger.debug(`Video download delay: ${difference} seconds.`); + logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); if (err) { logger.error(err.stderr); @@ -1195,7 +1188,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } // registers file in DB - file_uid = registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), 'video'); + file_uid = registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type); if (file_name) file_names.push(file_name); } @@ -1262,14 +1255,13 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { video.on('info', function(info) { video_info = info; file_size = video_info.size; - console.log('Download started') fs.writeJSONSync(removeFileExtension(video_info._filename) + '.info.json', video_info); video.pipe(fs.createWriteStream(video_info._filename, { flags: 'w' })) }); // Will be called if download was already completed and there is nothing more to download. video.on('complete', function complete(info) { 'use strict' - console.log('filename: ' + info._filename + ' already downloaded.') + logger.info('file ' + info._filename + ' already downloaded.') }) let download_pos = 0; @@ -1283,8 +1275,6 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { }); video.on('end', function() { - console.log('finished downloading!') - let new_date = Date.now(); let difference = (new_date - date)/1000; logger.debug(`Video download delay: ${difference} seconds.`); @@ -1292,8 +1282,12 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { download['complete'] = true; updateDownloads(); - // Does ID3 tagging if audio + // audio-only cleanup if (type === 'audio') { + // filename fix + video_info['_filename'] = removeFileExtension(video_info['_filename']) + '.mp3'; + + // ID3 tagging let tags = { title: video_info['title'], artist: video_info['artist'] ? video_info['artist'] : video_info['uploader'] @@ -1377,10 +1371,11 @@ async function generateArgs(url, type, options) { if (customOutput) { downloadConfig = ['-o', fileFolderPath + customOutput + "", qualityPath, '--write-info-json', '--print-json']; } else { - downloadConfig = ['-o', fileFolderPath + videopath + (is_audio ? '.mp3' : '.mp4'), qualityPath, '--write-info-json', '--print-json']; + downloadConfig = ['-o', fileFolderPath + videopath + (is_audio ? '.%(ext)s' : '.mp4'), qualityPath, '--write-info-json', '--print-json']; } if (is_audio) { + downloadConfig.push('-x'); downloadConfig.push('--audio-format', 'mp3'); } @@ -1695,7 +1690,7 @@ app.post('/api/tomp3', async function(req, res) { } const is_playlist = url.includes('playlist'); - if (is_playlist) + if (true || is_playlist) result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID); else result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID); From b58330594094f1462f55ec40e5819a4fc3931525 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Tue, 21 Apr 2020 20:10:53 -0400 Subject: [PATCH 6/7] Downloads in the download manager now get updated smoothly, preventing the DOM from updating on object reassign --- .../downloads/downloads.component.ts | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index f07d8d7..0cefc39 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -34,7 +34,7 @@ import { trigger, transition, animateChild, stagger, query, style, animate } fro export class DownloadsComponent implements OnInit { downloads_check_interval = 500; - downloads = null; + downloads = {}; keys = Object.keys; @@ -52,10 +52,7 @@ export class DownloadsComponent implements OnInit { getCurrentDownloads() { this.postsService.getCurrentDownloads().subscribe(res => { if (res['downloads']) { - if (JSON.stringify(this.downloads) !== JSON.stringify(res['downloads'])) { - // if they're not the same, then replace - this.downloads = res['downloads']; - } + this.assignNewValues(res['downloads']); } else { // failed to get downloads } @@ -89,6 +86,34 @@ export class DownloadsComponent implements OnInit { }); } + assignNewValues(new_downloads_by_session) { + const session_keys = Object.keys(new_downloads_by_session); + for (let i = 0; i < session_keys.length; i++) { + const session_id = session_keys[i]; + const session_downloads_by_id = new_downloads_by_session[session_id]; + const session_download_ids = Object.keys(session_downloads_by_id); + + if (!this.downloads[session_id]) { + this.downloads[session_id] = session_downloads_by_id; + } else { + for (let j = 0; j < session_download_ids.length; j++) { + const download_id = session_download_ids[j]; + const download = new_downloads_by_session[session_id][download_id] + if (!this.downloads[session_id][download_id]) { + this.downloads[session_id][download_id] = download; + } else { + const download_to_update = this.downloads[session_id][download_id]; + download_to_update['percent_complete'] = download['percent_complete']; + download_to_update['complete'] = download['complete']; + download_to_update['timestamp_end'] = download['timestamp_end']; + download_to_update['downloading'] = download['downloading']; + download_to_update['error'] = download['error']; + } + } + } + } + } + downloadsValid() { let valid = false; const keys = this.keys(this.downloads); From eca06a7fb19274e67cdef02fc0ef449e97571607 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Wed, 22 Apr 2020 21:42:21 -0400 Subject: [PATCH 7/7] Downloads on the home page now show the progress bar --- backend/app.js | 32 ++++++++++++++++++++++++++++++-- src/app/main/main.component.html | 6 +++--- src/app/main/main.component.ts | 32 ++++++++++++++++++++++++++++++-- src/app/posts.services.ts | 15 +++++++++++---- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/backend/app.js b/backend/app.js index ef87782..98facb1 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1117,6 +1117,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { if (!downloads[session]) downloads[session] = {}; downloads[session][download_uid] = { uid: download_uid, + ui_uid: options.ui_uid, downloading: true, complete: false, url: url, @@ -1231,6 +1232,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { if (!downloads[session]) downloads[session] = {}; downloads[session][download_uid] = { uid: download_uid, + ui_uid: options.ui_uid, downloading: true, complete: false, url: url, @@ -1686,7 +1688,8 @@ app.post('/api/tomp3', async function(req, res) { maxBitrate: req.body.maxBitrate, customQualityConfiguration: req.body.customQualityConfiguration, youtubeUsername: req.body.youtubeUsername, - youtubePassword: req.body.youtubePassword + youtubePassword: req.body.youtubePassword, + ui_uid: req.body.ui_uid } const is_playlist = url.includes('playlist'); @@ -1711,7 +1714,8 @@ app.post('/api/tomp4', async function(req, res) { selectedHeight: req.body.selectedHeight, customQualityConfiguration: req.body.customQualityConfiguration, youtubeUsername: req.body.youtubeUsername, - youtubePassword: req.body.youtubePassword + youtubePassword: req.body.youtubePassword, + ui_uid: req.body.ui_uid } const is_playlist = url.includes('playlist'); @@ -2443,6 +2447,30 @@ app.get('/api/audio/:id', function(req , res){ res.send({downloads: downloads}); }); + app.post('/api/download', async (req, res) => { + var session_id = req.body.session_id; + var download_id = req.body.download_id; + let found_download = null; + + // find download + if (downloads[session_id] && Object.keys(downloads[session_id])) { + let session_downloads = Object.values(downloads[session_id]); + for (let i = 0; i < session_downloads.length; i++) { + let session_download = session_downloads[i]; + if (session_download && session_download['ui_uid'] === download_id) { + found_download = session_download; + break; + } + } + } + + if (found_download) { + res.send({download: found_download}); + } else { + res.send({download: null}); + } + }); + app.post('/api/clearDownloads', async (req, res) => { let success = false; var delete_all = req.body.delete_all; diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index 84fe86e..46079fa 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -170,11 +170,11 @@
-
- +
+
-