diff --git a/backend/consts.js b/backend/consts.js index 5392b4e..ba081c0 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -134,14 +134,6 @@ exports.CONFIG_ITEMS = { 'key': 'ytdl_subscriptions_check_interval', 'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval' }, - 'ytdl_subscriptions_check_interval': { - 'key': 'ytdl_subscriptions_check_interval', - 'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval' - }, - 'ytdl_subscriptions_download_delay': { - 'key': 'ytdl_subscriptions_download_delay', - 'path': 'YoutubeDLMaterial.Subscriptions.download_delay' - }, 'ytdl_subscriptions_redownload_fresh_uploads': { 'key': 'ytdl_subscriptions_redownload_fresh_uploads', 'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads' diff --git a/backend/db.js b/backend/db.js index 9a30012..a4c8080 100644 --- a/backend/db.js +++ b/backend/db.js @@ -155,51 +155,17 @@ exports._connectToDB = async (custom_connection_string = null) => { } } -exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => { - let db_path = null; - const file_id = utils.removeFileExtension(file_path); - if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); - if (!file_object) { - logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); - return false; - } - - utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path); - - // add thumbnail path - file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path); - - // if category exists, only include essential info - if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; - - // modify duration - if (cropFileSettings) { - file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; - } - - if (multiUserMode) file_object['user_uid'] = multiUserMode.user; - - const file_obj = await registerFileDBManual(file_object); - - // remove metadata JSON if needed - if (!config_api.getConfigItem('ytdl_include_metadata')) { - utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path) - } - - return file_obj; -} - -exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { - if (!file_object) file_object = generateFileObject2(file_path, type); +exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { + if (!file_object) file_object = generateFileObject(file_path, type); if (!file_object) { logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); return false; } - utils.fixVideoMetadataPerms2(file_path, type); + utils.fixVideoMetadataPerms(file_path, type); // add thumbnail path - file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type); + file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path); // if category exists, only include essential info if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; @@ -216,7 +182,7 @@ exports.registerFileDB2 = async (file_path, type, user_uid = null, category = nu // remove metadata JSON if needed if (!config_api.getConfigItem('ytdl_include_metadata')) { - utils.deleteJSONFile2(file_path, type) + utils.deleteJSONFile(file_path, type) } return file_obj; @@ -234,36 +200,7 @@ async function registerFileDBManual(file_object) { return file_object; } -function generateFileObject(id, type, customPath = null, sub = null) { - if (!customPath && sub) { - customPath = getAppendedBasePathSub(sub, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true); - if (!jsonobj) { - return null; - } - const ext = (type === 'audio') ? '.mp3' : '.mp4' - const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); - // console. - var stats = fs.statSync(path.join(__dirname, file_path)); - - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A'; - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = type === 'audio'; - var description = jsonobj.description; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); - return file_obj; -} - -function generateFileObject2(file_path, type) { +function generateFileObject(file_path, type) { var jsonobj = utils.getJSON(file_path, type); if (!jsonobj) { return null; @@ -380,7 +317,7 @@ exports.importUnregisteredFiles = async () => { const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); if (!file_is_registered) { // add additional info - await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); + await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); logger.verbose(`Added discovered file to the database: ${file.id}`); } } @@ -388,24 +325,6 @@ exports.importUnregisteredFiles = async () => { } -exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => { - const preimported_file_paths = []; - - const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type); - for (let i = 0; i < files.length; i++) { - const file = files[i]; - // check if file exists in db, if not add it - const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id}); - if (!file_is_registered) { - // add additional info - await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file); - preimported_file_paths.push(file['path']); - logger.verbose(`Preemptively added subscription file to the database: ${file.id}`); - } - } - return preimported_file_paths; -} - exports.addMetadataPropertyToDB = async (property_key) => { try { const dirs_to_check = await exports.getFileDirectoriesAndDBs(); diff --git a/backend/downloader.js b/backend/downloader.js index dd07dab..e7fb9f3 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -194,9 +194,7 @@ async function collectInfo(download_uid) { let info = await getVideoInfoByURL(url, args, download_uid); if (!info) { - // info failed, record error and pause download - const error = 'Failed to get info, see server logs for specific error.'; - await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); + // info failed, error presumably already recorded return; } @@ -315,7 +313,7 @@ async function downloadQueuedFile(download_uid) { fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); logger.info('Renamed ' + file_name + '.webm to ' + file_name); } catch(e) { - + logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`); } } @@ -333,7 +331,7 @@ async function downloadQueuedFile(download_uid) { } // registers file in DB - const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); + const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); file_objs.push(file_obj); } @@ -544,9 +542,9 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) { } resolve(outputs.length === 1 ? outputs[0] : outputs); } catch(e) { - logger.error(`Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`); + const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`; + logger.error(error); if (download_uid) { - const error = 'Failed to get info, see server logs for specific error.'; await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); } resolve(null); @@ -558,7 +556,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) { } if (download_uid) { const error = 'Failed to get info, see server logs for specific error.'; - await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {error: err.stderr ? err.stderr : error, paused: true, running: false}); } resolve(null); } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 385c5fd..244cb70 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -3,7 +3,6 @@ const path = require('path'); const youtubedl = require('youtube-dl'); const config_api = require('./config'); -const twitch_api = require('./twitch'); const utils = require('./utils'); const logger = require('./logger'); diff --git a/backend/utils.js b/backend/utils.js index d258b4f..9c07833 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -144,24 +144,7 @@ function getJSONByType(type, name, customPath, openReadPerms = false) { return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) } -function getDownloadedThumbnail(name, type, customPath = null) { - if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path'); - - let jpgPath = path.join(customPath, name + '.jpg'); - let webpPath = path.join(customPath, name + '.webp'); - let pngPath = path.join(customPath, name + '.png'); - - if (fs.existsSync(jpgPath)) - return jpgPath; - else if (fs.existsSync(webpPath)) - return webpPath; - else if (fs.existsSync(pngPath)) - return pngPath; - else - return null; -} - -function getDownloadedThumbnail2(file_path, type) { +function getDownloadedThumbnail(file_path) { const file_path_no_extension = removeFileExtension(file_path); let jpgPath = file_path_no_extension + '.jpg'; @@ -199,29 +182,7 @@ function getExpectedFileSize(input_info_jsons) { return expected_filesize; } -function fixVideoMetadataPerms(name, type, customPath = null) { - if (is_windows) return; - if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') - : config_api.getConfigItem('ytdl_video_folder_path'); - - const ext = type === 'audio' ? '.mp3' : '.mp4'; - - const files_to_fix = [ - // JSONs - path.join(customPath, name + '.info.json'), - path.join(customPath, name + ext + '.info.json'), - // Thumbnails - path.join(customPath, name + '.webp'), - path.join(customPath, name + '.jpg') - ]; - - for (const file of files_to_fix) { - if (!fs.existsSync(file)) continue; - fs.chmodSync(file, 0o644); - } -} - -function fixVideoMetadataPerms2(file_path, type) { +function fixVideoMetadataPerms(file_path, type) { if (is_windows) return; const ext = type === 'audio' ? '.mp3' : '.mp4'; @@ -243,19 +204,7 @@ function fixVideoMetadataPerms2(file_path, type) { } } -function deleteJSONFile(name, type, customPath = null) { - if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') - : config_api.getConfigItem('ytdl_video_folder_path'); - - const ext = type === 'audio' ? '.mp3' : '.mp4'; - let json_path = path.join(customPath, name + '.info.json'); - let alternate_json_path = path.join(customPath, name + ext + '.info.json'); - - if (fs.existsSync(json_path)) fs.unlinkSync(json_path); - if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path); -} - -function deleteJSONFile2(file_path, type) { +function deleteJSONFile(file_path, type) { const ext = type === 'audio' ? '.mp3' : '.mp4'; const file_path_no_extension = removeFileExtension(file_path); @@ -291,7 +240,6 @@ async function removeIDFromArchive(archive_path, id) { const updatedData = dataArray.join('\n'); await fs.writeFile(archive_path, updatedData); if (line) return line; - if (err) throw err; } function durationStringToNumber(dur_str) { @@ -429,12 +377,9 @@ module.exports = { getJSON: getJSON, getTrueFileName: getTrueFileName, getDownloadedThumbnail: getDownloadedThumbnail, - getDownloadedThumbnail2: getDownloadedThumbnail2, getExpectedFileSize: getExpectedFileSize, fixVideoMetadataPerms: fixVideoMetadataPerms, - fixVideoMetadataPerms2: fixVideoMetadataPerms2, deleteJSONFile: deleteJSONFile, - deleteJSONFile2: deleteJSONFile2, removeIDFromArchive: removeIDFromArchive, getDownloadedFilesByType: getDownloadedFilesByType, createContainerZipFile: createContainerZipFile,